feat: support ${resources.xx.info.xx}, ${components.xx.output.xx}

.github/workflows/ci.yml

@@ -59,3 +59,61 @@ jobs:
 
       - name: Run tests
         run: npm run ci
+  # publish-beta:
+  #   runs-on: ubuntu-latest
+
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       package:
+  #         - component-interface
+  #         - component-manager
+  #         - credential
+  #         - diff
+  #         - downloads
+  #         - engine
+  #         - ignore-walk
+  #         - load-application
+  #         - load-component
+  #         - logger
+  #         - orm
+  #         - parse-spec
+  #         - progress-bar
+  #         - registry
+  #         - utils
+  #         - zip
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #     with:
+  #       fetch-depth: 2
+  #   - name: Set up Node.js
+  #     uses: actions/setup-node@v2
+  #     with:
+  #       node-version: '18'
+  #       registry-url: https://registry.npmjs.org/
+
+  #   - name: Install pnpm
+  #     run: npm install -g pnpm
+
+  #   - name: Install dependencies
+  #     run: npm run install:all
+
+  #   - name: Build packages
+  #     run: npm run build
+
+  #   - name: Publish package
+  #     run: |
+  #       PACKAGE_DIR=packages/${{ matrix.package }}
+  #       PACKAGE_JSON="${PACKAGE_DIR}/package.json"
+
+  #       # 检查 package.json 是否被修改
+  #       if git diff HEAD^ HEAD --name-only | grep -q "${PACKAGE_JSON}"; then
+  #         echo "Publishing ${{ matrix.package }}..."
+  #         cd $PACKAGE_DIR
+  #         pnpm publish --tag=beta
+  #       else
+  #         echo "No changes in ${{ matrix.package }}/package.json"
+  #       fi
+  #     env:
+  #       NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}

packages/component-interface/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/component-interface",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "request for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/component-interface/src/index.ts

@@ -21,4 +21,5 @@ export interface IInputs {
   args: string[];
   cwd: string;
   outputs?: Record<string, any>;
+  output?: Record<string, any>; // 当前步骤输出
 }

packages/credential/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/credential",
-  "version": "0.0.7",
+  "version": "0.0.9-beta.1",
   "description": "credential for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/credential/src/constant.ts

@@ -1,6 +1,5 @@
 import path from 'path';
 import os from 'os';
-import * as setType from './actions/set/type';
 
 const Crypto = require('crypto-js');
 

packages/engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/engine",
-  "version": "0.1.3",
+  "version": "0.1.4-beta.9",
   "description": "a engine lib for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/engine/src/actions/index.ts

@@ -1,7 +1,7 @@
 import { IAction, IActionLevel, IActionType, IAllowFailure, IComponentAction, IHookType, IPluginAction, IRunAction, getInputs } from '@serverless-devs/parse-spec';
 import { isEmpty, filter, includes, set, get } from 'lodash';
 import * as utils from '@serverless-devs/utils';
-import { DevsError, ETrackerType } from '@serverless-devs/utils';
+import { DevsError, ETrackerType, isDevsDebugMode } from '@serverless-devs/utils';
 import fs from 'fs-extra';
 import { spawn } from 'child_process';
 import loadComponent from '@serverless-devs/load-component';
@@ -12,7 +12,7 @@ import { ILoggerInstance } from '@serverless-devs/logger';
 import { EXIT_CODE } from '../constants';
 import { IStepOptions } from '../types';
 
-const debug = require('@serverless-cd/debug')('serverless-devs:engine');
+const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:engine') : (i: any) => {};
 
 interface IRecord {
   magic: Record<string, any>; // 记录魔法变量
@@ -284,7 +284,7 @@ You can still use them now, but we suggest to modify them.`)
     const [componentName, command] = _;
 
     // Load the specified component.
-    const instance = await loadComponent(componentName, { logger: this.logger });
+    const instance = await loadComponent(componentName, { logger: this.logger, cleanCache: true });
 
     // Check if the specified command exists for the component.
     if (instance[command]) {

packages/engine/src/constants/index.ts

@@ -8,3 +8,7 @@ export const EXIT_CODE = {
   // shell 执行错误
   RUN: 101,
 };
+
+export const INFO_EXP_PATTERN = /\$\{resources\.([^.{}]+)\.info(\.[^.{}]+)+}/g;
+export const COMPONENT_EXP_PATTERN = /\$\{components\.([^.{}]+)\.output(\.[^.{}]+)+}/g;
+export const OUTPUT_EXP_PATTERN = /\$\{resources\.([^.{}]+)\.output(\.[^.{}]+)+}/g;

packages/engine/src/index.ts

@@ -1,5 +1,5 @@
 import { createMachine, interpret } from 'xstate';
-import { isEmpty, get, each, map, isFunction, has, uniqueId, filter, omit, includes, set, isNil, isUndefined, keys, size } from 'lodash';
+import { isEmpty, get, each, map, isFunction, has, uniqueId, filter, omit, includes, set, isNil, isUndefined, keys, size, cloneDeep, find } from 'lodash';
 import { IStepOptions, IRecord, IStatus, IEngineOptions, IContext, IEngineError, STEP_STATUS } from './types';
 import { getProcessTime, getCredential, stringify, getAllowFailure } from './utils';
 import ParseSpec, { getInputs, ISpec, IHookType, IStep as IParseStep, IActionLevel } from '@serverless-devs/parse-spec';
@@ -9,14 +9,14 @@ import Actions from './actions';
 import Credential from '@serverless-devs/credential';
 import loadComponent from '@serverless-devs/load-component';
 import Logger, { ILoggerInstance } from '@serverless-devs/logger';
-import { DevsError, ETrackerType, emoji, getAbsolutePath, getRootHome, getUserAgent, traceid } from '@serverless-devs/utils';
-import { EXIT_CODE } from './constants';
+import { DevsError, ETrackerType, emoji, getAbsolutePath, getRootHome, getUserAgent, traceid, isDevsDebugMode } from '@serverless-devs/utils';
+import { EXIT_CODE, INFO_EXP_PATTERN, COMPONENT_EXP_PATTERN } from './constants';
 import assert from 'assert';
 import Ajv from 'ajv';
 export * from './types';
 export { verify, preview, init } from './utils';
 
-const debug = require('@serverless-cd/debug')('serverless-devs:engine');
+const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:engine') : (i: any) => {};
 
 /**
  * Engine Class
@@ -38,6 +38,7 @@ class Engine {
   private parseSpecInstance!: ParseSpec;
   private globalActionInstance!: Actions; // 全局的 action
   private actionInstance!: Actions; // 项目的 action
+  private info: Record<string, any> = {}; // 存储全局变量
 
   constructor(private options: IEngineOptions) {
     debug('engine start');
@@ -65,10 +66,11 @@ class Engine {
     each(this.options.env, (value, key) => {
       process.env[key] = value;
     });
-    const { steps: _steps } = this.spec;
+    const { steps: _steps, allSteps } = this.spec;
     // 参数校验
     await this.validate();
     this.context.steps = await this.download(_steps);
+    this.context.allSteps = allSteps ? await this.download(allSteps) : [];
   }
 
   /**
@@ -103,7 +105,7 @@ class Engine {
     this.context.credential = credential;
     // 处理 global-pre
     try {
-      this.globalActionInstance.setValue('magic', this.getFilterContext());
+      this.globalActionInstance.setValue('magic', await this.getFilterContext());
       this.globalActionInstance.setValue('command', command);
       await this.globalActionInstance.start(IHookType.PRE, { access, credential });
     } catch (error) {
@@ -117,6 +119,9 @@ class Engine {
     this.context.steps = map(this.context.steps, item => {
       return { ...item, stepCount: uniqueId(), status: STEP_STATUS.PENDING, done: false };
     });
+    this.context.allSteps = map(this.context.allSteps, item => {
+      return { ...item, stepCount: uniqueId(), status: STEP_STATUS.PENDING, done: false };
+    });
     const res: IContext = await new Promise(async resolve => {
       // Every states object has two fixed states, "init" and "final".
       const states: any = {
@@ -329,21 +334,77 @@ class Engine {
     });
   }
 
+  /**
+   * 20240624 New feature
+   * ${resources.xx.info.xx} read from `info` command's output.
+   * This function will check current step content to determine 
+   * whether to execute `info` command of a certain resource.
+   * 
+   * @param item - The current step being processed.
+   * @returns
+   */
+  private async getInfo(item: IStepOptions | undefined) {
+    if (!item) return;
+    const { props } = item;
+    const compString = JSON.stringify(props);
+    const matches = compString.matchAll(INFO_EXP_PATTERN);
+    const resourceNames = new Set<string>();
+    for (const match of matches) {
+      if (match[1]) {
+        resourceNames.add(match[1]);
+      }
+    }
+    // support ${components.xx.output.xx}
+    const comMatches = compString.matchAll(COMPONENT_EXP_PATTERN);
+    for (const match of comMatches) {
+      if (match[1]) {
+        resourceNames.add(match[1]);
+      }
+    }
+    if (isEmpty(resourceNames)) return;
+    const resourcesItems: IStepOptions[] = map(Array.from(resourceNames), (name) => {
+      return this.context.allSteps.find((obj) => obj.projectName === name) as IStepOptions;
+    });
+    await Promise.all(map(resourcesItems, async (item) => {
+      if (!item) return;
+      const projectInfo = get(this.info, item.projectName);
+      if (!projectInfo || isEmpty(projectInfo) || isNil(projectInfo)) {
+        this.logger.write(`${chalk.gray(`execute info command of [${item.projectName}]...`)}`);
+        const spec = cloneDeep(this.spec);
+        spec['command'] = 'info';
+        const res: any = await this.doSrc(item, {}, spec);
+        set(this.info, item.projectName, res);
+        this.logger.write(`${chalk.gray(`[${item.projectName}] info command executed.`)}`);
+      }
+    }));
+  }
   /**
    * Generates a context data for the given step containing details like cwd, vars, and other steps' outputs and props.
    * @param item - The current step being processed.
    * @returns - The generated context data.
    */
-  private getFilterContext(item?: IStepOptions) {
+  private async getFilterContext(item?: IStepOptions) {
+    await this.getInfo(item);
     const data = {
       cwd: path.dirname(this.spec.yaml.path),
       vars: this.spec.yaml.vars,
       resources: {},
+      components: {},
       __runtime: this.options.verify ? 'engine' : 'parse',
       __steps: this.context.steps,
     } as Record<string, any>;
-    for (const obj of this.context.steps) {
-      data.resources[obj.projectName] = { output: obj.output || {}, props: obj.props || {} };
+    for (const obj of this.context.allSteps) {
+      const stepItem = find(this.context.steps, (obj2) => obj2.projectName === obj.projectName);
+      data.resources[obj.projectName] = { 
+        output: obj.output || get(stepItem, 'output') || {}, 
+        props: obj.props || get(stepItem, 'props') || {}, 
+        info: this.info[obj.projectName] || {},
+        vars: obj.vars || get(stepItem, 'vars') || {},
+      };
+      // support ${components.xx.output.xx}
+      data.components[obj.projectName] = {
+        output: this.info[obj.projectName] || {},
+      }
     }
     if (item) {
       data.credential = item.credential;
@@ -353,6 +414,7 @@ class Engine {
         component: item.component,
         props: data.resources[item.projectName].props,
         output: data.resources[item.projectName].output,
+        vars: data.resources[item.projectName].vars,
       };
     }
     return data;
@@ -419,7 +481,7 @@ class Engine {
       // project success hook
       try {
         // 项目的output, 再次获取魔法变量
-        this.actionInstance.setValue('magic', this.getFilterContext(item));
+        this.actionInstance.setValue('magic', await this.getFilterContext(item));
         const res = await this.actionInstance?.start(IHookType.SUCCESS, {
           ...this.record.componentProps,
           output: get(item, 'output', {}),
@@ -488,7 +550,7 @@ class Engine {
       });
 
       // Set values for the action instance.
-      this.actionInstance.setValue('magic', this.getFilterContext(item));
+      this.actionInstance.setValue('magic', await this.getFilterContext(item));
       this.actionInstance.setValue('step', item);
       this.actionInstance.setValue('command', command);
 
@@ -559,7 +621,7 @@ class Engine {
    * @returns An object containing properties related to the project step.
    */
   private async getProps(item: IStepOptions) {
-    const magic = this.getFilterContext(item);
+    const magic = await this.getFilterContext(item);
     debug(`magic context: ${JSON.stringify(magic)}`);
     const newInputs = getInputs(item.props, magic);
     const { projectName, command } = this.spec;
@@ -595,9 +657,9 @@ class Engine {
    * @param data - Additional data which may contain plugin output.
    * @returns Result of the executed action, if applicable.
    */
-  private async doSrc(item: IStepOptions, data: Record<string, any> = {}) {
+  private async doSrc(item: IStepOptions, data: Record<string, any> = {}, spec = this.spec) {
     // Extract command and projectName from the specification.
-    const { command = '', projectName, yaml } = this.spec;
+    const { command = '', projectName, yaml } = spec;
 
     // Retrieve properties for the given project step.
     const newInputs = await this.getProps(item);

packages/engine/src/types.ts

@@ -73,6 +73,7 @@ export interface IContext {
   error: IEngineError[]; // 记录step的错误信息
   output: Record<string, any>; // 记录step的输出
   credential: Record<string, any>; // 尝试获取到的密钥信息
+  allSteps: IStepOptions[]; // 记录所有step
 }
 
 export type IEngineError = Error | AssertionError | DevsError;

packages/load-application/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/load-application",
-  "version": "0.0.13",
+  "version": "0.0.14-beta.5",
   "description": "load application for serverless-devs",
   "main": "lib/index.js",
   "scripts": {
@@ -18,12 +18,12 @@
   },
   "dependencies": {
     "@serverless-cd/debug": "^4.3.4",
-    "@serverless-devs/art-template": "^4.13.15",
+    "@serverless-devs/art-template": "^4.13.16-beta.12",
     "@serverless-devs/credential": "workspace:^",
     "@serverless-devs/downloads": "workspace:^",
     "@serverless-devs/utils": "workspace:^",
     "art-template": "^4.13.2",
-    "axios": "^1.4.0",
+    "axios": "^1.6.0",
     "chalk": "^4.1.2",
     "fs-extra": "^11.1.0",
     "inquirer": "^8.2.4",

packages/load-application/src/index.ts

@@ -4,7 +4,8 @@ import assert from 'assert';
 import { IOptions } from './types';
 import { includes, get } from 'lodash';
 import { REGISTRY } from './constant';
-const debug = require('@serverless-cd/debug')('serverless-devs:load-appliaction');
+import { isDevsDebugMode } from '@serverless-devs/utils';
+const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:load-application') : (i: any) => {};
 
 export default async (template: string, options: IOptions = {}) => {
   debug(`load application, template: ${template}, options: ${JSON.stringify(options)}`);

packages/load-application/src/v2.ts

@@ -3,19 +3,18 @@ import fs from 'fs-extra';
 import axios from 'axios';
 import download from '@serverless-devs/downloads';
 import artTemplate from 'art-template';
-import { getYamlContent, isCiCdEnvironment, getYamlPath } from '@serverless-devs/utils';
+import { getYamlContent, isCiCdEnvironment, getYamlPath, isDevsDebugMode } from '@serverless-devs/utils';
 import { isEmpty, includes, split, get, has, set, sortBy, map, concat, keys, find, startsWith } from 'lodash';
 import parse from './parse';
 import { IProvider, IOptions } from './types';
 import { CONFIGURE_LATER, DEFAULT_MAGIC_ACCESS, REGISTRY } from './constant';
-import { getInputs, getAllCredential, getDefaultValue } from './utils';
-import YAML from 'yaml';
+import { getAllCredential, getDefaultValue } from './utils';
 import inquirer from 'inquirer';
 import chalk from 'chalk';
 import Credential from '@serverless-devs/credential';
-import { gray, RANDOM_PATTERN } from './constant';
+import { gray } from './constant';
 import assert from 'assert';
-const debug = require('@serverless-cd/debug')('serverless-devs:load-appliaction');
+const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:load-application') : (i: any) => {};
 
 class LoadApplication {
   private provider: `${IProvider}`;
@@ -163,7 +162,6 @@ class LoadApplication {
     const hookPath = path.join(this.tempPath, 'hook');
     if (!fs.existsSync(hookPath)) return;
     const { logger } = this.options;
-    const { parameters, access } = this.options;
     const hook = await require(hookPath);
     const data = {
       provider: this.provider,