Skip to content

Commit

Permalink
Add benchmark program
Browse files Browse the repository at this point in the history
  • Loading branch information
samchon committed Jun 17, 2024
1 parent d6ed8eb commit 1eee1e4
Show file tree
Hide file tree
Showing 10 changed files with 178 additions and 8 deletions.
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,10 @@ npm run test -- --include cart order issue
npm run test -- --include cart order issue --exclude index deposit
```

For reference, if you run `npm run benchmark` command, your test functions defined in the [test/features/api](test/features/api) directory would be utilized for performance benchmarking. If you want to see the performance bench result earlier, visit below link please:

- [docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md](https://github.com/samchon/backend/blob/master/docs/benchmarks/AMD%20Ryzen%209%207940HS%20w%20Radeon%20780M%20Graphics.md)

### 3.4. Main Program
After [Definition](#31-definition), client [SDK](#32-software-development-kit) building and [Test Automation Program](#33-test-automation-program) are all prepared, finally you can develop the Main Program. Also, when you complete the Main Program implementation, it would better to validate the implementation through the pre-built [SDK](#32-software-development-kit) and [Test Automation Program](#33-test-automation-program).

Expand All @@ -202,6 +206,7 @@ List of the run commands defined in the [package.json](package.json) are like be

- Test
- **`test`**: **Run [Test Automation Program](#33-test-automation-program)**
- `benchmark`: Run performance benchmark program
- Build
- `build`: Build every below programs
- `build:sdk`: Build SDK library, but only for local
Expand Down Expand Up @@ -233,6 +238,7 @@ List of the run commands defined in the [package.json](package.json) are like be
- [src/providers/](src/providers/): Service providers (bridge between DB and controllers)
- [src/executable/](src/executable/): Executable programs
- [**test/**](test/): Test Automation Program
- [test/benchmark](test/benchmark): Performance Benchmark Program

### 4.3. Related Repositories
> Write the related repositories down.
25 changes: 25 additions & 0 deletions docs/benchmarks/AMD Ryzen 9 7940HS w Radeon 780M Graphics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
## Benchmark Report
> - CPU: AMD Ryzen 9 7940HS w/ Radeon 780M Graphics
> - RAM: 31 GB
> - NodeJS Version: v20.10.0
> - Backend Server: 1 core / 1 thread
> - Arguments:
> - Count: 1,024
> - Threads: 4
> - Simultaneous: 32
> - Total Elapsed Time: 501 ms
### Total
Type | Count | Success | Mean. | Stdev. | Minimum | Maximum
----|----|----|----|----|----|----
Total | 1,052 | 1,052 | 14.08 | 7.8 | 3 | 74

### Endpoints
Type | Count | Success | Mean. | Stdev. | Minimum | Maximum
----|----|----|----|----|----|----
GET /monitors/system | 563 | 563 | 14.12 | 7.78 | 3 | 71
GET /monitors/health | 489 | 489 | 14.04 | 7.83 | 4 | 74

### Failures
Method | Path | Count | Success
-------|------|-------|--------
11 changes: 6 additions & 5 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"version": "0.1.0",
"description": "Backend for PROJECT",
"scripts": {
"benchmark": "rimraf prisma/migrations && node bin/test/benchmark",
"test": "rimraf prisma/migrations && node bin/test",
"------------------------BUILDS------------------------": "",
"build": "npm run build:prisma && npm run build:sdk && npm run build:main && npm run build:test",
Expand Down Expand Up @@ -38,8 +39,8 @@
},
"homepage": "https://github.com/samchon/backend",
"devDependencies": {
"@nestia/e2e": "^0.4.3",
"@nestia/sdk": "^3.1.1",
"@nestia/e2e": "^0.5.0",
"@nestia/sdk": "^3.2.2",
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
"@types/cli": "^0.11.19",
"@types/express": "^4.17.21",
Expand Down Expand Up @@ -74,8 +75,8 @@
"write-file-webpack-plugin": "^4.5.1"
},
"dependencies": {
"@nestia/core": "^3.1.1",
"@nestia/fetcher": "^3.1.1",
"@nestia/core": "^3.2.2",
"@nestia/fetcher": "^3.2.2",
"@nestjs/common": "^10.3.8",
"@nestjs/core": "^10.3.8",
"@nestjs/platform-express": "^10.3.8",
Expand All @@ -88,7 +89,7 @@
"serialize-error": "^4.1.0",
"source-map-support": "^0.5.19",
"tstl": "^3.0.0",
"typia": "^6.0.3"
"typia": "^6.1.0"
},
"private": true
}
4 changes: 2 additions & 2 deletions packages/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
},
"homepage": "https://github.com/samchon/backend#readme",
"dependencies": {
"@nestia/fetcher": "^3.1.1",
"typia": "^6.0.3"
"@nestia/fetcher": "^3.2.2",
"typia": "^6.1.0"
},
"include": [
"lib",
Expand Down
1 change: 1 addition & 0 deletions src/api/functional/monitors/health/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export async function get(connection: IConnection): Promise<void> {
? get.simulate(connection)
: PlainFetcher.fetch(connection, {
...get.METADATA,
template: get.METADATA.path,
path: get.path(),
});
}
Expand Down
1 change: 1 addition & 0 deletions src/api/functional/monitors/performance/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export async function get(connection: IConnection): Promise<get.Output> {
? get.simulate(connection)
: PlainFetcher.fetch(connection, {
...get.METADATA,
template: get.METADATA.path,
path: get.path(),
});
}
Expand Down
3 changes: 3 additions & 0 deletions src/api/functional/monitors/system/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export async function get(connection: IConnection): Promise<get.Output> {
? get.simulate(connection)
: PlainFetcher.fetch(connection, {
...get.METADATA,
template: get.METADATA.path,
path: get.path(),
});
}
Expand Down Expand Up @@ -72,6 +73,7 @@ export async function internal_server_error(
? internal_server_error.simulate(connection)
: PlainFetcher.fetch(connection, {
...internal_server_error.METADATA,
template: internal_server_error.METADATA.path,
path: internal_server_error.path(),
});
}
Expand Down Expand Up @@ -114,6 +116,7 @@ export async function uncaught_exception(
? uncaught_exception.simulate(connection)
: PlainFetcher.fetch(connection, {
...uncaught_exception.METADATA,
template: uncaught_exception.METADATA.path,
path: uncaught_exception.path(),
});
}
Expand Down
17 changes: 16 additions & 1 deletion src/utils/ArgumentParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export namespace ArgumentParser {
message: string,
) => <Choice extends string>(choices: Choice[]) => Promise<Choice>;
boolean: (name: string) => (message: string) => Promise<boolean>;
number: (name: string) => (message: string) => Promise<number>;
}

export const parse = async <T>(
Expand Down Expand Up @@ -57,10 +58,24 @@ export namespace ArgumentParser {
message,
})
)[name] as boolean;
const number = (name: string) => async (message: string) =>
Number(
(
await inquirer.createPromptModule()({
type: "number",
name,
message,
})
)[name],
);

const output: T | Error = await (async () => {
try {
return await inquiry(commander.program, { select, boolean }, action);
return await inquiry(
commander.program,
{ select, boolean, number },
action,
);
} catch (error) {
return error as Error;
}
Expand Down
103 changes: 103 additions & 0 deletions test/benchmark/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
import { DynamicBenchmarker } from "@nestia/e2e";
import fs from "fs";
import os from "os";

import { MyBackend } from "../../src/MyBackend";
import { MyConfiguration } from "../../src/MyConfiguration";
import { MyGlobal } from "../../src/MyGlobal";
import { ArgumentParser } from "../../src/utils/ArgumentParser";

interface IOptions {
include?: string[];
exclude?: string[];
count: number;
threads: number;
simultaneous: number;
}

const getOptions = () =>
ArgumentParser.parse<IOptions>(async (command, prompt, action) => {
// command.option("--mode <string>", "target mode");
// command.option("--reset <true|false>", "reset local DB or not");
command.option("--include <string...>", "include feature files");
command.option("--exclude <string...>", "exclude feature files");
command.option("--count <number>", "number of requests to make");
command.option("--threads <number>", "number of threads to use");
command.option(
"--simultaneous <number>",
"number of simultaneous requests to make",
);
return action(async (options) => {
// if (typeof options.reset === "string")
// options.reset = options.reset === "true";
// options.mode ??= await prompt.select("mode")("Select mode")([
// "LOCAL",
// "DEV",
// "REAL",
// ]);
// options.reset ??= await prompt.boolean("reset")("Reset local DB");
options.count = Number(
options.count ??
(await prompt.number("count")("Number of requests to make")),
);
options.threads = Number(
options.threads ??
(await prompt.number("threads")("Number of threads to use")),
);
options.simultaneous = Number(
options.simultaneous ??
(await prompt.number("simultaneous")(
"Number of simultaneous requests to make",
)),
);
return options as IOptions;
});
});

const main = async (): Promise<void> => {
// CONFIGURATIONS
const options: IOptions = await getOptions();
MyGlobal.testing = true;

// BACKEND SERVER
const backend: MyBackend = new MyBackend();
await backend.open();

// DO BENCHMARK
const report: DynamicBenchmarker.IReport = await DynamicBenchmarker.master({
count: options.count,
threads: options.threads,
simultaneous: options.simultaneous,
filter: (func) =>
(!options.include?.length ||
(options.include ?? []).some((str) => func.includes(str))) &&
(!options.exclude?.length ||
(options.exclude ?? []).every((str) => !func.includes(str))),
servant: `${__dirname}/servant.js`,
});

// DOCUMENTATION
try {
await fs.promises.mkdir(`${MyConfiguration.ROOT}/docs/benchmarks`, {
recursive: true,
});
} catch {}
await fs.promises.writeFile(
`${MyConfiguration.ROOT}/docs/benchmarks/${os
.cpus()[0]
.model.trim()
.split("\\")
.join("")
.split("/")
.join("")}.md`,
DynamicBenchmarker.markdown(report),
"utf8",
);

// CLOSE
await backend.close();
};
main().catch((exp) => {
console.error(exp);
process.exit(-1);
});
15 changes: 15 additions & 0 deletions test/benchmark/servant.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { DynamicBenchmarker } from "@nestia/e2e";

import { MyConfiguration } from "../../src/MyConfiguration";

DynamicBenchmarker.servant({
connection: {
host: `http://127.0.0.1:${MyConfiguration.API_PORT()}`,
},
location: `${__dirname}/../features`,
parameters: (connection) => [connection],
prefix: "test_api_",
}).catch((exp) => {
console.error(exp);
process.exit(-1);
});

0 comments on commit 1eee1e4

Please sign in to comment.