fix: improve developer tools (#4119)

extensions/README.md

@@ -0,0 +1,9 @@
+## The `extensions` Folder
+
+The `extensions` folder serves as the central location for creating Webiny extensions.
+
+To quickly start working on a new extension, in your terminal, simply run the 
+`yarn webiny scaffold` command. Once ready, from the list of available scaffolds, 
+choose **New Extension** and then proceed by answering the rest of the questions.
+
+For more information, please visit https://webiny.link/extensions.

packages/cli-plugin-deploy-pulumi/commands/deploy/executeDeploy.js

@@ -1,5 +1,6 @@
 const { measureDuration } = require("../../utils");
 const ora = require("ora");
+const isCI = require("is-ci");
 
 const spinnerMessages = [
     [60, "Still deploying..."],
@@ -22,7 +23,7 @@ const spinnerMessages = [
 
 module.exports = async ({ inputs, context, pulumi }) => {
     // We always show deployment logs when doing previews.
-    const showDeploymentLogs = inputs.deploymentLogs;
+    const showDeploymentLogs = isCI || inputs.deploymentLogs;
 
     const PULUMI_SECRETS_PROVIDER = process.env.PULUMI_SECRETS_PROVIDER;
     const PULUMI_CONFIG_PASSPHRASE = process.env.PULUMI_CONFIG_PASSPHRASE;

packages/cli-plugin-deploy-pulumi/commands/index.js

@@ -54,9 +54,50 @@ module.exports = [
                     });
                     yargs.option("deployment-logs", {
                         default: undefined,
-                        describe: `Print deployment logs`,
+                        describe: `Print deployment logs (automatically enabled in CI environments)`,
                         type: "boolean"
                     });
+
+                    yargs
+                        .option("allow-local-state-files", {
+                            describe: `Allow using local Pulumi state files with production environment deployment (not recommended).`,
+                            type: "boolean"
+                        })
+                        .check(args => {
+                            const { red } = require("chalk");
+                            const { env, allowLocalStateFiles } = args;
+
+                            // If the folder is not defined, we are destroying the whole project.
+                            // In that case, we must confirm the environment name to destroy.
+                            const prodEnvs = ["prod", "production"];
+                            const isProdEnv = prodEnvs.includes(env);
+                            if (!isProdEnv) {
+                                return true;
+                            }
+
+                            let pulumiBackend =
+                                process.env.WEBINY_PULUMI_BACKEND ||
+                                process.env.WEBINY_PULUMI_BACKEND_URL ||
+                                process.env.PULUMI_LOGIN;
+
+                            if (pulumiBackend) {
+                                return true;
+                            }
+
+                            if (allowLocalStateFiles) {
+                                return true;
+                            }
+
+                            throw new Error(
+                                [
+                                    "Please confirm you want to use local Pulumi state files with",
+                                    "your production deployment by appending",
+                                    `${red(
+                                        "--allow-local-state-files"
+                                    )} to the command. Learn more: https://webiny.link/state-files-production.`
+                                ].join(" ")
+                            );
+                        });
                 },
                 async argv => {
                     return require("./deploy")(argv, context);
@@ -99,7 +140,7 @@ module.exports = [
 
             yargs.command(
                 "watch [folder]",
-                `Rebuild and deploy specified specified project application while making changes to it`,
+                `Rebuild and deploy specified project application while making changes to it`,
                 yargs => {
                     yargs.example("$0 watch api --env=dev");
                     yargs.example(

packages/cli-plugin-deploy-pulumi/package.json

@@ -21,6 +21,7 @@
     "express": "^4.19.2",
     "fast-glob": "^3.2.7",
     "humanize-duration": "^3.31.0",
+    "is-ci": "^3.0.0",
     "listr": "^0.14.3",
     "localtunnel": "2.0.2",
     "lodash": "^4.17.21",

packages/cli-plugin-scaffold-extensions/src/generators/admin.ts

@@ -1,6 +1,21 @@
 import { addPluginToReactApp } from "./utils/addPluginToReactApp";
 import { PluginGenerator } from "~/types";
+import path from "path";
+import readJson from "load-json-file";
+import { PackageJson } from "@webiny/cli-plugin-scaffold/types";
+import writeJson from "write-json-file";
 
 export const adminGenerator: PluginGenerator = async ({ input }) => {
     await addPluginToReactApp(input);
+
+    // Update dependencies list in package.json.
+    const packageJsonPath = path.join("apps", "admin", "package.json");
+    const packageJson = await readJson<PackageJson>(packageJsonPath);
+    if (!packageJson.dependencies) {
+        packageJson.dependencies = {};
+    }
+
+    packageJson.dependencies[input.packageName] = "1.0.0";
+
+    await writeJson(packageJsonPath, packageJson);
 };

packages/cli-plugin-scaffold-extensions/src/generators/api.ts

@@ -1,6 +1,21 @@
 import { addPluginToApiApp } from "./utils/addPluginToApiApp";
 import { PluginGenerator } from "~/types";
+import path from "path";
+import readJson from "load-json-file";
+import { PackageJson } from "@webiny/cli-plugin-scaffold/types";
+import writeJson from "write-json-file";
 
 export const apiGenerator: PluginGenerator = async ({ input }) => {
     await addPluginToApiApp(input);
+
+    // Update dependencies list in package.json.
+    const packageJsonPath = path.join("apps", "api", "graphql", "package.json");
+    const packageJson = await readJson<PackageJson>(packageJsonPath);
+    if (!packageJson.dependencies) {
+        packageJson.dependencies = {};
+    }
+
+    packageJson.dependencies[input.packageName] = "1.0.0";
+
+    await writeJson(packageJsonPath, packageJson);
 };

packages/cli-plugin-scaffold-extensions/src/index.ts

@@ -24,6 +24,7 @@ interface Input {
     name: string;
     packageName: string;
     location: string;
+    dependencies?: string;
 }
 
 const EXTENSIONS_ROOT_FOLDER = "extensions";
@@ -63,25 +64,6 @@ export default (): CliCommandScaffoldTemplate<Input> => ({
                         return true;
                     }
                 },
-                {
-                    name: "packageName",
-                    message: "Enter the package name:",
-                    default: (answers: Input) => {
-                        return Case.kebab(answers.name);
-                    },
-                    validate: pkgName => {
-                        if (!pkgName) {
-                            return "Missing package name.";
-                        }
-
-                        const isValidName = validateNpmPackageName(pkgName);
-                        if (!isValidName) {
-                            return `Package name must look something like "my-custom-extension".`;
-                        }
-
-                        return true;
-                    }
-                },
                 {
                     name: "location",
                     message: `Enter the extension location:`,
@@ -102,12 +84,31 @@ export default (): CliCommandScaffoldTemplate<Input> => ({
                             return `The target location already exists "${location}".`;
                         }
 
+                        return true;
+                    }
+                },
+                {
+                    name: "packageName",
+                    message: "Enter the package name:",
+                    default: (answers: Input) => {
+                        return Case.kebab(answers.name);
+                    },
+                    validate: pkgName => {
+                        if (!pkgName) {
+                            return "Missing package name.";
+                        }
+
+                        const isValidName = validateNpmPackageName(pkgName);
+                        if (!isValidName) {
+                            return `Package name must be a valid NPM package name, for example "my-custom-extension".`;
+                        }
+
                         return true;
                     }
                 }
             ];
         },
-        generate: async ({ input, ora }) => {
+        generate: async ({ input, ora, context }) => {
             const { type, name } = input;
             if (!type) {
                 throw new Error("Missing extension type.");
@@ -158,6 +159,43 @@ export default (): CliCommandScaffoldTemplate<Input> => ({
 
                 replaceInPath(path.join(location, "**/*.*"), codeReplacements);
 
+                if (input.dependencies) {
+                    const packageJsonPath = path.join(location, "package.json");
+                    const packageJson = await readJson<PackageJson>(packageJsonPath);
+                    if (!packageJson.dependencies) {
+                        packageJson.dependencies = {};
+                    }
+
+                    const packages = input.dependencies.split(",");
+                    for (const packageName of packages) {
+                        const isWebinyPackage = packageName.startsWith("@webiny/");
+                        if (isWebinyPackage) {
+                            packageJson.dependencies[packageName] = context.version;
+                            continue;
+                        }
+
+                        try {
+                            const { stdout } = await execa("npm", [
+                                "view",
+                                packageName,
+                                "version",
+                                "json"
+                            ]);
+
+                            packageJson.dependencies[packageName] = `^${stdout}`;
+                        } catch (e) {
+                            throw new Error(
+                                `Could not find ${log.red.hl(
+                                    packageName
+                                )} NPM package. Please double-check the package name and try again.`,
+                                { cause: e }
+                            );
+                        }
+                    }
+
+                    await writeJson(packageJsonPath, packageJson);
+                }
+
                 // Add package to workspaces
                 const rootPackageJsonPath = path.join(project.root, "package.json");
                 const rootPackageJson = await readJson<PackageJson>(rootPackageJsonPath);
@@ -170,7 +208,12 @@ export default (): CliCommandScaffoldTemplate<Input> => ({
                     await generators[type]({ input: { name, packageName } });
                 }
 
-                // Once everything is done, run `yarn` so the new packages are automatically installed.
+                // Sleep for 1 second before proceeding with yarn installation.
+                await new Promise(resolve => {
+                    setTimeout(resolve, 1000);
+                });
+
+                // Once everything is done, run `yarn` so the new packages are installed.
                 await execa("yarn");
 
                 ora.succeed(`New extension created in ${log.success.hl(location)}.`);

packages/cwp-template-aws/template/common/extensions/README.md

@@ -0,0 +1,9 @@
+## The `extensions` Folder
+
+The `extensions` folder serves as the central location for creating Webiny extensions.
+
+To quickly start working on a new extension, in your terminal, simply run the
+`yarn webiny scaffold` command. Once ready, from the list of available scaffolds,
+choose **New Extension** and then proceed by answering the rest of the questions.
+
+For more information, please visit https://webiny.link/extensions.

packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts

@@ -21,6 +21,7 @@ import {
     withCommonLambdaEnvVariables,
     withServiceManifest
 } from "~/utils";
+import { DEFAULT_PROD_ENV_NAMES } from "~/constants";
 
 export type ApiPulumiApp = ReturnType<typeof createApiPulumiApp>;
 
@@ -126,7 +127,8 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams =
                 });
             }
 
-            const productionEnvironments = app.params.create.productionEnvironments || ["prod"];
+            const productionEnvironments =
+                app.params.create.productionEnvironments || DEFAULT_PROD_ENV_NAMES;
             const isProduction = productionEnvironments.includes(app.params.run.env);
 
             // Enables logs forwarding.
@@ -261,7 +263,6 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams =
                 apwSchedulerEventRule: apwScheduler.eventRule.output.name,
                 apwSchedulerEventTargetId: apwScheduler.eventTarget.output.targetId,
                 dynamoDbTable: core.primaryDynamodbTableName,
-                dynamoDbElasticsearchTable: core.elasticsearchDynamodbTableName,
                 migrationLambdaArn: migration.function.output.arn,
                 graphqlLambdaName: graphql.functions.graphql.output.name,
                 backgroundTaskLambdaArn: backgroundTask.backgroundTask.output.arn,
@@ -270,6 +271,13 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams =
                 websocketApiUrl: websocket.websocketApiUrl
             });
 
+            // Only add `dynamoDbElasticsearchTable` output if using search engine (ES/OS).
+            if (searchEngineParams) {
+                app.addOutputs({
+                    dynamoDbElasticsearchTable: core.elasticsearchDynamodbTableName
+                });
+            }
+
             app.addHandler(() => {
                 addDomainsUrlsOutputs({
                     app,

packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts

@@ -16,7 +16,7 @@ import {
 
 import { getAwsAccountId } from "../awsUtils";
 import { CoreVpc } from "./CoreVpc";
-import { LAMBDA_RUNTIME } from "~/constants";
+import { DEFAULT_PROD_ENV_NAMES, LAMBDA_RUNTIME } from "~/constants";
 
 export interface ElasticSearchParams {
     protect: boolean;
@@ -46,7 +46,8 @@ export const ElasticSearch = createAppModule({
         const domainName = "webiny-js";
         const accountId = getAwsAccountId(app);
 
-        const productionEnvironments = app.params.create.productionEnvironments || ["prod"];
+        const productionEnvironments =
+            app.params.create.productionEnvironments || DEFAULT_PROD_ENV_NAMES;
         const isProduction = productionEnvironments.includes(app.params.run.env);
 
         const vpc = app.getModule(CoreVpc, { optional: true });

packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts

@@ -17,7 +17,7 @@ import {
 
 import { getAwsAccountId } from "../awsUtils";
 import { CoreVpc } from "./CoreVpc";
-import { LAMBDA_RUNTIME } from "~/constants";
+import { DEFAULT_PROD_ENV_NAMES, LAMBDA_RUNTIME } from "~/constants";
 
 export interface OpenSearchParams {
     protect: boolean;
@@ -46,7 +46,8 @@ const OS_ENGINE_VERSION = "OpenSearch_2.11";
 export const OpenSearch = createAppModule({
     name: "OpenSearch",
     config(app, params: OpenSearchParams) {
-        const productionEnvironments = app.params.create.productionEnvironments || ["prod"];
+        const productionEnvironments =
+            app.params.create.productionEnvironments || DEFAULT_PROD_ENV_NAMES;
         const isProduction = productionEnvironments.includes(app.params.run.env);
 
         const vpc = app.getModule(CoreVpc, { optional: true });

packages/pulumi-aws/src/apps/core/createCorePulumiApp.ts

@@ -10,6 +10,7 @@ import { CoreVpc } from "./CoreVpc";
 import { tagResources } from "~/utils";
 import { withServiceManifest } from "~/utils/withServiceManifest";
 import { addServiceManifestTableItem, TableDefinition } from "~/utils/addServiceManifestTableItem";
+import { DEFAULT_PROD_ENV_NAMES } from "~/constants";
 
 export type CorePulumiApp = ReturnType<typeof createCorePulumiApp>;
 
@@ -131,7 +132,8 @@ export function createCorePulumiApp(projectAppParams: CreateCorePulumiAppParams
                 });
             }
 
-            const productionEnvironments = app.params.create.productionEnvironments || ["prod"];
+            const productionEnvironments =
+                app.params.create.productionEnvironments || DEFAULT_PROD_ENV_NAMES;
             const isProduction = productionEnvironments.includes(app.params.run.env);
 
             const protect = app.getParam(projectAppParams.protect) ?? isProduction;

packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts

@@ -9,6 +9,7 @@ import { CoreOutput, VpcConfig } from "~/apps";
 import { addDomainsUrlsOutputs, tagResources, withCommonLambdaEnvVariables } from "~/utils";
 import { applyTenantRouter } from "~/apps/tenantRouter";
 import { withServiceManifest } from "~/utils/withServiceManifest";
+import { DEFAULT_PROD_ENV_NAMES } from "~/constants";
 
 export type WebsitePulumiApp = ReturnType<typeof createWebsitePulumiApp>;
 
@@ -73,7 +74,8 @@ export const createWebsitePulumiApp = (projectAppParams: CreateWebsitePulumiAppP
                 });
             }
 
-            const productionEnvironments = app.params.create.productionEnvironments || ["prod"];
+            const productionEnvironments =
+                app.params.create.productionEnvironments || DEFAULT_PROD_ENV_NAMES;
             const isProduction = productionEnvironments.includes(app.params.run.env);
 
             // Register core output as a module available for all other modules

packages/pulumi-aws/src/constants.ts

@@ -1,3 +1,5 @@
 import { lambda } from "@pulumi/aws";
 
 export const LAMBDA_RUNTIME = lambda.Runtime.NodeJS18dX;
+
+export const DEFAULT_PROD_ENV_NAMES = ["prod", "production"];