From 1eee1e4d469992a7f5ce601b1fd3ab81cbcddb5f Mon Sep 17 00:00:00 2001 From: Jeongho Nam Date: Tue, 18 Jun 2024 00:02:36 +0900 Subject: [PATCH] Add benchmark program --- README.md | 6 + ...D Ryzen 9 7940HS w Radeon 780M Graphics.md | 25 +++++ package.json | 11 +- packages/api/package.json | 4 +- src/api/functional/monitors/health/index.ts | 1 + .../functional/monitors/performance/index.ts | 1 + src/api/functional/monitors/system/index.ts | 3 + src/utils/ArgumentParser.ts | 17 ++- test/benchmark/index.ts | 103 ++++++++++++++++++ test/benchmark/servant.ts | 15 +++ 10 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md create mode 100644 test/benchmark/index.ts create mode 100644 test/benchmark/servant.ts diff --git a/README.md b/README.md index 805f906..5fad8c0 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,10 @@ npm run test -- --include cart order issue npm run test -- --include cart order issue --exclude index deposit ``` +For reference, if you run `npm run benchmark` command, your test functions defined in the [test/features/api](test/features/api) directory would be utilized for performance benchmarking. If you want to see the performance bench result earlier, visit below link please: + + - [docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md](https://github.com/samchon/backend/blob/master/docs/benchmarks/AMD%20Ryzen%209%207940HS%20w%20Radeon%20780M%20Graphics.md) + ### 3.4. Main Program After [Definition](#31-definition), client [SDK](#32-software-development-kit) building and [Test Automation Program](#33-test-automation-program) are all prepared, finally you can develop the Main Program. Also, when you complete the Main Program implementation, it would better to validate the implementation through the pre-built [SDK](#32-software-development-kit) and [Test Automation Program](#33-test-automation-program). @@ -202,6 +206,7 @@ List of the run commands defined in the [package.json](package.json) are like be - Test - **`test`**: **Run [Test Automation Program](#33-test-automation-program)** + - `benchmark`: Run performance benchmark program - Build - `build`: Build every below programs - `build:sdk`: Build SDK library, but only for local @@ -233,6 +238,7 @@ List of the run commands defined in the [package.json](package.json) are like be - [src/providers/](src/providers/): Service providers (bridge between DB and controllers) - [src/executable/](src/executable/): Executable programs - [**test/**](test/): Test Automation Program + - [test/benchmark](test/benchmark): Performance Benchmark Program ### 4.3. Related Repositories > Write the related repositories down. \ No newline at end of file diff --git a/docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md b/docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md new file mode 100644 index 0000000..5fba8fc --- /dev/null +++ b/docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md @@ -0,0 +1,25 @@ +## Benchmark Report +> - CPU: AMD Ryzen 9 7940HS w/ Radeon 780M Graphics +> - RAM: 31 GB +> - NodeJS Version: v20.10.0 +> - Backend Server: 1 core / 1 thread +> - Arguments: +> - Count: 1,024 +> - Threads: 4 +> - Simultaneous: 32 +> - Total Elapsed Time: 501 ms + +### Total +Type | Count | Success | Mean. | Stdev. | Minimum | Maximum +----|----|----|----|----|----|---- +Total | 1,052 | 1,052 | 14.08 | 7.8 | 3 | 74 + +### Endpoints +Type | Count | Success | Mean. | Stdev. | Minimum | Maximum +----|----|----|----|----|----|---- +GET /monitors/system | 563 | 563 | 14.12 | 7.78 | 3 | 71 +GET /monitors/health | 489 | 489 | 14.04 | 7.83 | 4 | 74 + +### Failures +Method | Path | Count | Success +-------|------|-------|-------- \ No newline at end of file diff --git a/package.json b/package.json index f83329f..1853941 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.1.0", "description": "Backend for PROJECT", "scripts": { + "benchmark": "rimraf prisma/migrations && node bin/test/benchmark", "test": "rimraf prisma/migrations && node bin/test", "------------------------BUILDS------------------------": "", "build": "npm run build:prisma && npm run build:sdk && npm run build:main && npm run build:test", @@ -38,8 +39,8 @@ }, "homepage": "https://github.com/samchon/backend", "devDependencies": { - "@nestia/e2e": "^0.4.3", - "@nestia/sdk": "^3.1.1", + "@nestia/e2e": "^0.5.0", + "@nestia/sdk": "^3.2.2", "@trivago/prettier-plugin-sort-imports": "^4.3.0", "@types/cli": "^0.11.19", "@types/express": "^4.17.21", @@ -74,8 +75,8 @@ "write-file-webpack-plugin": "^4.5.1" }, "dependencies": { - "@nestia/core": "^3.1.1", - "@nestia/fetcher": "^3.1.1", + "@nestia/core": "^3.2.2", + "@nestia/fetcher": "^3.2.2", "@nestjs/common": "^10.3.8", "@nestjs/core": "^10.3.8", "@nestjs/platform-express": "^10.3.8", @@ -88,7 +89,7 @@ "serialize-error": "^4.1.0", "source-map-support": "^0.5.19", "tstl": "^3.0.0", - "typia": "^6.0.3" + "typia": "^6.1.0" }, "private": true } diff --git a/packages/api/package.json b/packages/api/package.json index 889b8b0..5dba8d2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -15,8 +15,8 @@ }, "homepage": "https://github.com/samchon/backend#readme", "dependencies": { - "@nestia/fetcher": "^3.1.1", - "typia": "^6.0.3" + "@nestia/fetcher": "^3.2.2", + "typia": "^6.1.0" }, "include": [ "lib", diff --git a/src/api/functional/monitors/health/index.ts b/src/api/functional/monitors/health/index.ts index e6cbcb3..28935c1 100644 --- a/src/api/functional/monitors/health/index.ts +++ b/src/api/functional/monitors/health/index.ts @@ -23,6 +23,7 @@ export async function get(connection: IConnection): Promise { ? get.simulate(connection) : PlainFetcher.fetch(connection, { ...get.METADATA, + template: get.METADATA.path, path: get.path(), }); } diff --git a/src/api/functional/monitors/performance/index.ts b/src/api/functional/monitors/performance/index.ts index 3d5226d..49aa30f 100644 --- a/src/api/functional/monitors/performance/index.ts +++ b/src/api/functional/monitors/performance/index.ts @@ -28,6 +28,7 @@ export async function get(connection: IConnection): Promise { ? get.simulate(connection) : PlainFetcher.fetch(connection, { ...get.METADATA, + template: get.METADATA.path, path: get.path(), }); } diff --git a/src/api/functional/monitors/system/index.ts b/src/api/functional/monitors/system/index.ts index 15fc5a5..744c610 100644 --- a/src/api/functional/monitors/system/index.ts +++ b/src/api/functional/monitors/system/index.ts @@ -28,6 +28,7 @@ export async function get(connection: IConnection): Promise { ? get.simulate(connection) : PlainFetcher.fetch(connection, { ...get.METADATA, + template: get.METADATA.path, path: get.path(), }); } @@ -72,6 +73,7 @@ export async function internal_server_error( ? internal_server_error.simulate(connection) : PlainFetcher.fetch(connection, { ...internal_server_error.METADATA, + template: internal_server_error.METADATA.path, path: internal_server_error.path(), }); } @@ -114,6 +116,7 @@ export async function uncaught_exception( ? uncaught_exception.simulate(connection) : PlainFetcher.fetch(connection, { ...uncaught_exception.METADATA, + template: uncaught_exception.METADATA.path, path: uncaught_exception.path(), }); } diff --git a/src/utils/ArgumentParser.ts b/src/utils/ArgumentParser.ts index d23a4be..b4dd48a 100644 --- a/src/utils/ArgumentParser.ts +++ b/src/utils/ArgumentParser.ts @@ -15,6 +15,7 @@ export namespace ArgumentParser { message: string, ) => (choices: Choice[]) => Promise; boolean: (name: string) => (message: string) => Promise; + number: (name: string) => (message: string) => Promise; } export const parse = async ( @@ -57,10 +58,24 @@ export namespace ArgumentParser { message, }) )[name] as boolean; + const number = (name: string) => async (message: string) => + Number( + ( + await inquirer.createPromptModule()({ + type: "number", + name, + message, + }) + )[name], + ); const output: T | Error = await (async () => { try { - return await inquiry(commander.program, { select, boolean }, action); + return await inquiry( + commander.program, + { select, boolean, number }, + action, + ); } catch (error) { return error as Error; } diff --git a/test/benchmark/index.ts b/test/benchmark/index.ts new file mode 100644 index 0000000..31ed3d9 --- /dev/null +++ b/test/benchmark/index.ts @@ -0,0 +1,103 @@ +import { DynamicBenchmarker } from "@nestia/e2e"; +import fs from "fs"; +import os from "os"; + +import { MyBackend } from "../../src/MyBackend"; +import { MyConfiguration } from "../../src/MyConfiguration"; +import { MyGlobal } from "../../src/MyGlobal"; +import { ArgumentParser } from "../../src/utils/ArgumentParser"; + +interface IOptions { + include?: string[]; + exclude?: string[]; + count: number; + threads: number; + simultaneous: number; +} + +const getOptions = () => + ArgumentParser.parse(async (command, prompt, action) => { + // command.option("--mode ", "target mode"); + // command.option("--reset ", "reset local DB or not"); + command.option("--include ", "include feature files"); + command.option("--exclude ", "exclude feature files"); + command.option("--count ", "number of requests to make"); + command.option("--threads ", "number of threads to use"); + command.option( + "--simultaneous ", + "number of simultaneous requests to make", + ); + return action(async (options) => { + // if (typeof options.reset === "string") + // options.reset = options.reset === "true"; + // options.mode ??= await prompt.select("mode")("Select mode")([ + // "LOCAL", + // "DEV", + // "REAL", + // ]); + // options.reset ??= await prompt.boolean("reset")("Reset local DB"); + options.count = Number( + options.count ?? + (await prompt.number("count")("Number of requests to make")), + ); + options.threads = Number( + options.threads ?? + (await prompt.number("threads")("Number of threads to use")), + ); + options.simultaneous = Number( + options.simultaneous ?? + (await prompt.number("simultaneous")( + "Number of simultaneous requests to make", + )), + ); + return options as IOptions; + }); + }); + +const main = async (): Promise => { + // CONFIGURATIONS + const options: IOptions = await getOptions(); + MyGlobal.testing = true; + + // BACKEND SERVER + const backend: MyBackend = new MyBackend(); + await backend.open(); + + // DO BENCHMARK + const report: DynamicBenchmarker.IReport = await DynamicBenchmarker.master({ + count: options.count, + threads: options.threads, + simultaneous: options.simultaneous, + filter: (func) => + (!options.include?.length || + (options.include ?? []).some((str) => func.includes(str))) && + (!options.exclude?.length || + (options.exclude ?? []).every((str) => !func.includes(str))), + servant: `${__dirname}/servant.js`, + }); + + // DOCUMENTATION + try { + await fs.promises.mkdir(`${MyConfiguration.ROOT}/docs/benchmarks`, { + recursive: true, + }); + } catch {} + await fs.promises.writeFile( + `${MyConfiguration.ROOT}/docs/benchmarks/${os + .cpus()[0] + .model.trim() + .split("\\") + .join("") + .split("/") + .join("")}.md`, + DynamicBenchmarker.markdown(report), + "utf8", + ); + + // CLOSE + await backend.close(); +}; +main().catch((exp) => { + console.error(exp); + process.exit(-1); +}); diff --git a/test/benchmark/servant.ts b/test/benchmark/servant.ts new file mode 100644 index 0000000..a64ab1a --- /dev/null +++ b/test/benchmark/servant.ts @@ -0,0 +1,15 @@ +import { DynamicBenchmarker } from "@nestia/e2e"; + +import { MyConfiguration } from "../../src/MyConfiguration"; + +DynamicBenchmarker.servant({ + connection: { + host: `http://127.0.0.1:${MyConfiguration.API_PORT()}`, + }, + location: `${__dirname}/../features`, + parameters: (connection) => [connection], + prefix: "test_api_", +}).catch((exp) => { + console.error(exp); + process.exit(-1); +});