Merge pull request #581 from samchon/features/is

Complement #574 and #578 - new benchmark program

README.md

@@ -37,7 +37,7 @@ export function prune<T extends object>(input: T): void; // erase extra props
 
 All functions in `typia` require **only one line**. You don't need any extra dedication like JSON schema definitions or decorator function calls. Just call `typia` function with only one line like `typia.assert<T>(input)`.
 
-Also, as `typia` performs AOT (Ahead of Time) compilation skill, its performance is much faster than other competitive libaries. For an example, when comparing validate function `is()` with other competitive libraries, `typia` is maximum **15,000x faster** than `class-validator`.
+Also, as `typia` performs AOT (Ahead of Time) compilation skill, its performance is much faster than other competitive libaries. For an example, when comparing validate function `is()` with other competitive libraries, `typia` is maximum **20,000x faster** than `class-validator`.
 
 ![Is Function Benchmark](https://github.com/samchon/typia/raw/master/benchmark/results/11th%20Gen%20Intel(R)%20Core(TM)%20i5-1135G7%20%40%202.40GHz/images/is.svg)
 
@@ -250,7 +250,7 @@ export function createAssertStringify<T>(): (input: T) => string;
   - `application()`: generate JSON schema with only one line
     - you can complement JSON schema contents through [comment tags](https://github.com/samchon/typia/wiki/Enhanced-JSON#comment-tags)
   - `assertParse()`: parse JSON string safely with type validation
-  - `isStringify()`: maximum 10x faster JSON stringify fuction even type safe
+  - `isStringify()`: maximum 160x faster JSON stringify fuction even type safe
 
 ![JSON string conversion speed](https://raw.githubusercontent.com/samchon/typia/master/benchmark/results/AMD%20Ryzen%207%206800HS%20with%20Radeon%20Graphics/images/stringify.svg)
 
@@ -286,8 +286,8 @@ If you need specific random data generation, utilize comment tags or do customiz
 [Nestia](https://github.com/samchon/nestia) is a set of helper libraries for `NestJS`, supporting below features:
 
   - `@nestia/core`: superfast decorators using `typia`
-    - **15,000x faster** validation
-    - **100x faster** JSON serialization
+    - **20,000x faster** validation
+    - **200x faster** JSON serialization
   - `@nestia/sdk`: evolved **SDK** and **Swagger** generators
     - SDK (Software Development Kit)
       - interaction library for client developers

benchmark/generate/assert-error.ts

@@ -0,0 +1,84 @@
+import { BenchmarkProgrammer } from "./BenchmarkProgrammer";
+
+const FEATURES: string[] = [
+    "ObjectSimple",
+    "ObjectHierarchical",
+    "ObjectRecursive",
+    "ObjectUnionExplicit",
+    "ObjectUnionImplicit",
+    "ArrayRecursive",
+    "ArrayRecursiveUnionExplicit",
+    "ArrayRecursiveUnionImplicit",
+    "UltimateUnion",
+];
+
+const LIBRARIES = (category: string): BenchmarkProgrammer.ILibrary[] => [
+    {
+        name: "typia",
+        body: (type) => {
+            const program = `create${BenchmarkProgrammer.pascal(
+                category,
+            )}BenchmarkProgram`;
+            return [
+                `import typia from "typia";`,
+                ``,
+                `import { ${type} } from "../../../../test/structures/${type}";`,
+                `import { ${program} } from "../${program}";`,
+                ``,
+                `${program}(`,
+                `    typia.create${BenchmarkProgrammer.pascal(category).replace(
+                    "Error",
+                    "",
+                )}<${BenchmarkProgrammer.pascal(type)}[]>()`,
+                `);`,
+            ].join("\n");
+        },
+    },
+    {
+        name: "typebox",
+        body: (type) => {
+            const schema: string = `Typebox${type}`;
+            const program: string = `create${BenchmarkProgrammer.pascal(
+                category,
+            )}TypeboxBenchmarkProgram`;
+
+            return [
+                `import { __${schema} } from "../../../structures/typebox/${schema}";`,
+                ``,
+                `import { ${program} } from "./${program}";`,
+                ``,
+                `${program}(__${schema});`,
+            ].join("\n");
+        },
+    },
+    ...["io-ts", "zod", "class-validator"].map((name) => ({
+        name,
+        body: (type: string) => {
+            const schema: string = `${BenchmarkProgrammer.pascal(name)}${type}`;
+            const program: string = `create${BenchmarkProgrammer.pascal(
+                category,
+            )}${BenchmarkProgrammer.pascal(name)}BenchmarkProgram`;
+
+            return [
+                `import { ${schema} } from "../../../structures/${name}/${schema}";`,
+                ``,
+                `import { ${program} } from "./${program}";`,
+                ``,
+                `${program}(${schema});`,
+            ].join("\n");
+        },
+    })),
+];
+
+async function main(): Promise<void> {
+    await BenchmarkProgrammer.generate({
+        name: "assert-error",
+        libraries: LIBRARIES("assert-error"),
+        features: FEATURES,
+    });
+}
+
+main().catch((exp) => {
+    console.error(exp);
+    process.exit(-1);
+});

benchmark/generate/index.ts

@@ -2,3 +2,4 @@ import "./optimizer";
 import "./server";
 import "./stringify";
 import "./validate";
+import "./validate-error";

benchmark/generate/validate-error.ts

@@ -0,0 +1,89 @@
+import { BenchmarkProgrammer } from "./BenchmarkProgrammer";
+
+const FEATURES: string[] = [
+    "ObjectSimple",
+    "ObjectHierarchical",
+    "ObjectRecursive",
+    "ObjectUnionExplicit",
+    "ObjectUnionImplicit",
+    "ArrayRecursive",
+    "ArrayRecursiveUnionExplicit",
+    "ArrayRecursiveUnionImplicit",
+    "UltimateUnion",
+];
+
+const LIBRARIES = (category: string): BenchmarkProgrammer.ILibrary[] => [
+    {
+        name: "typia",
+        body: (type) => {
+            const program = `create${BenchmarkProgrammer.pascal(
+                category,
+            )}BenchmarkProgram`;
+            return [
+                `import typia from "typia";`,
+                ``,
+                `import { ${type} } from "../../../../test/structures/${type}";`,
+                `import { ${program} } from "../${program}";`,
+                ``,
+                `${program}(`,
+                `    typia.create${BenchmarkProgrammer.pascal(category).replace(
+                    "Error",
+                    "",
+                )}<${BenchmarkProgrammer.pascal(type)}[]>()`,
+                `);`,
+            ].join("\n");
+        },
+    },
+    {
+        name: "typebox",
+        body: (type) => {
+            const schema: string = `Typebox${type}`;
+            const program: string = `create${BenchmarkProgrammer.pascal(
+                category,
+            )}TypeboxBenchmarkProgram`;
+
+            return [
+                `import { __${schema} } from "../../../structures/typebox/${schema}";`,
+                ``,
+                `import { ${program} } from "./${program}";`,
+                ``,
+                `${program}(__${schema});`,
+            ].join("\n");
+        },
+    },
+    ...["io-ts", "zod", "class-validator"].map((name) => ({
+        name,
+        body: (type: string) => {
+            const schema: string = `${BenchmarkProgrammer.pascal(name)}${type}`;
+            const program: string = `create${BenchmarkProgrammer.pascal(
+                category,
+            )}${BenchmarkProgrammer.pascal(name)}BenchmarkProgram`;
+
+            return [
+                `import { ${schema} } from "../../../structures/${name}/${schema}";`,
+                ``,
+                `import { ${program} } from "./${program}";`,
+                ``,
+                `${program}(${schema});`,
+            ].join("\n");
+        },
+    })),
+];
+
+async function main(): Promise<void> {
+    await BenchmarkProgrammer.generate({
+        name: "assert-error",
+        libraries: LIBRARIES("assert-error"),
+        features: FEATURES,
+    });
+    await BenchmarkProgrammer.generate({
+        name: "validate-error",
+        libraries: LIBRARIES("validate-error"),
+        features: FEATURES,
+    });
+}
+
+main().catch((exp) => {
+    console.error(exp);
+    process.exit(-1);
+});

benchmark/index.ts

@@ -10,6 +10,8 @@ async function main(): Promise<void> {
         "is",
         "assert",
         "validate",
+        "assert-error",
+        "validate-error",
         "optimizer",
         "stringify",
         "server",

benchmark/internal/BenchmarkServer.ts

@@ -1,5 +1,6 @@
 import fs from "fs";
 import tgrid from "tgrid";
+import { Driver } from "tgrid/components/Driver";
 
 import { Spoiler } from "../../test/helpers/Spoiler";
 import { ArrayHierarchical } from "../../test/structures/ArrayHierarchical";
@@ -71,7 +72,7 @@ export namespace BenchmarkServer {
         for (const library of props.libraries) {
             const file: string = `${__dirname}/../programs/${props.category}/${library}/benchmark-${props.category}-${library}-${props.type}.${EXTENSION}`;
             result[library] = fs.existsSync(file)
-                ? await measureFunction(props.category)(props.type)(file)
+                ? await measureFunction(props.type)(file)
                 : null;
 
             console.log(`    - ${library}:`, result[library] !== null);
@@ -80,8 +81,7 @@ export namespace BenchmarkServer {
     };
 
     const measureFunction =
-        (category: string) =>
-        (type: string) =>
+        <T>(type: string) =>
         async (
             file: string,
         ): Promise<IBenchmarkProgram.IMeasurement | null> => {
@@ -93,28 +93,49 @@ export namespace BenchmarkServer {
             );
             await connector.connect(file);
 
-            const controller = connector.getDriver<IBenchmarkProgram<any>>();
+            const base = connector.getDriver<IBenchmarkProgram<T>>();
+            const result: IBenchmarkProgram.IMeasurement | null =
+                (await base.type()) === "success"
+                    ? await measureSuccess(type)(factory)(connector.getDriver())
+                    : await measureFailure(factory)(connector.getDriver());
+
+            await connector.close();
+            return result;
+        };
+
+    const measureSuccess =
+        (type: string) =>
+        <T>(factory: IFactory<T>) =>
+        async (controller: Driver<IBenchmarkProgram.ISuccessProgram<T>>) => {
+            const input: any = factory.generate();
             const tolerable = async () => {
+                if (await controller.skip(type)) return true;
                 for (const s of factory.SPOILERS) {
                     const fake: any = factory.generate();
                     s(fake);
-                    if ((await controller.validate(fake)) === true)
-                        return false;
+                    if (await controller.validate(fake)) return false;
                 }
-                return true;
+                return controller.validate(input);
             };
-            const input: any = factory.generate();
-            const result: IBenchmarkProgram.IMeasurement | null =
-                await (async () => {
-                    const success = () => controller.measure(input);
-                    if (await controller.skip(type)) return success();
-                    else if ((await controller.validate(input)) === false)
-                        return null;
-                    else if ((await tolerable()) === false) return null;
-                    return success();
-                })();
-            await connector.close();
-            return result;
+            return (await tolerable()) ? controller.success(input) : null;
+        };
+
+    const measureFailure =
+        <T>(factory: IFactory<T>) =>
+        async (controller: Driver<IBenchmarkProgram.IFailureProgram<T>>) => {
+            const data: T[] = [
+                ...new Array(99).fill(0).map(() => factory.generate()),
+                factory.trail(),
+            ];
+            const tolerable = async () => {
+                for (const s of factory.SPOILERS) {
+                    const fake: T = factory.generate();
+                    s(fake);
+                    if (await controller.validate([fake])) return false;
+                }
+                return controller.validate(data.slice(0, 99));
+            };
+            return (await tolerable()) ? controller.failure(data) : null;
         };
 
     async function findLibaries(path: string): Promise<string[]> {
@@ -145,6 +166,7 @@ export namespace BenchmarkServer {
 
     interface IFactory<T> {
         generate(): T;
+        trail(): T;
         SPOILERS: Spoiler<T>[];
     }
     const FACTORIES: Record<string, IFactory<any>> = {

benchmark/internal/BenchmarktReporter.ts

@@ -33,11 +33,12 @@ export namespace BenchmarkReporter {
                 const label: string = DICTIONARY[type];
                 const record: string[] = report.libraries.map((library) => {
                     const value = report.result[type][library];
-                    if (value === null || isNaN(value.amount)) return " - ";
+                    if (value === null) return " - ";
 
                     const space: number = Math.floor(
                         value.amount / value.time / 1_024,
                     );
+                    if (isNaN(space)) return " - ";
                     return space.toLocaleString();
                 });
                 await stream.write(` ${label} | ${record.join(" | ")} `);
@@ -75,7 +76,7 @@ export namespace BenchmarkReporter {
                 },
             );
 
-            const svg = HorizontalBarChart.generate(
+            const svg = HorizontalBarChart.generate(stream.environments)(
                 `${report.category} benchmark`,
             )(report.libraries)(relatives);
             await fs.promises.writeFile(
@@ -91,25 +92,30 @@ export namespace BenchmarkReporter {
                 ? `${__dirname}/../results`
                 : `${__dirname}/../../../benchmark/results`;
 
-        const memory: number = os.totalmem();
         const cpu: string = os.cpus()[0].model.trim();
         const location: string = `${results}/${cpu}`;
 
         await mkdir(results);
         await mkdir(location);
         await mkdir(`${location}/images`);
 
-        const stream: BenchmarkStream = new BenchmarkStream(location);
+        const stream: BenchmarkStream = new BenchmarkStream(location, {
+            cpu,
+            memory: os.totalmem(),
+            os: os.platform(),
+            node: process.version,
+            typia: await get_package_version(),
+        });
         await stream.write("# Benchmark of `typia`");
         await stream.write(`> - CPU: ${cpu}`);
         await stream.write(
             `> - Memory: ${Math.round(
-                memory / 1024 / 1024,
+                stream.environments.memory / 1024 / 1024,
             ).toLocaleString()} MB`,
         );
-        await stream.write(`> - OS: ${os.platform()}`);
-        await stream.write(`> - NodeJS version: ${process.version}`);
-        await stream.write(`> - Typia version: ${await get_package_version()}`);
+        await stream.write(`> - OS: ${stream.environments.os}`);
+        await stream.write(`> - NodeJS version: ${stream.environments.node}`);
+        await stream.write(`> - Typia version: v${stream.environments.typia}`);
         await stream.write("\n");
         return stream;
     }

benchmark/internal/BenhmarkStream.ts

@@ -4,7 +4,16 @@ export class BenchmarkStream {
     private readonly stream_: fs.WriteStream;
     private readonly time_: Date;
 
-    public constructor(public readonly path: string) {
+    public constructor(
+        public readonly path: string,
+        public readonly environments: {
+            cpu: string;
+            os: string;
+            memory: number;
+            node: string;
+            typia: string;
+        },
+    ) {
         this.stream_ = fs.createWriteStream(path + "/README.md", "utf8");
         this.time_ = new Date();
     }