feat(perftest-helpers): support mode analysis

actions/perftest-helpers/src/modules/sendReport.test.ts

@@ -40,15 +40,6 @@ describe('perftest/sendReport', () => {
             expect(gotMock).not.toHaveBeenCalled();
         });
 
-        it('should not send if result is empty', async () => {
-            readJsonMock.mockResolvedValue({ isVersionChanged: false, result: {} });
-
-            await new ApiWithoutTransform().send({ reportPath: 'azaza' } as any);
-
-            expect(transformMock).not.toHaveBeenCalled();
-            expect(gotMock).not.toHaveBeenCalled();
-        });
-
         it('should call this.transform if input is ok', async () => {
             const jsonReport = { isVersionChanged: false, result: { foo: 'bar' }, staticTaskChange: { baz: 'kl' } };
             const sendParams = {

actions/perftest-helpers/src/modules/sendReport.ts

@@ -5,7 +5,7 @@ import got from 'got';
 
 import { readJson } from '../utils/fs';
 
-import { ReportType, SubjectResult } from './types';
+import { ComparableResult, ReportType, SubjectResult } from './types';
 
 export type SendParams = {
     commitHash: string;
@@ -29,6 +29,8 @@ type MetricsRecord = {
 class PerftestSendReportApi {
     METRICS_URL = 'https://metrics.prom.third-party-app.sberdevices.ru/perftool';
 
+    MAX_BODY_SIZE = 1024 ** 2 / 2;
+
     async send({ reportPath, commitHash, subject, referrer }: SendParams): Promise<void> {
         const { staticTaskChange, result, isVersionChanged } = await readJson<ReportType>(path.resolve(reportPath));
 
@@ -38,12 +40,6 @@ class PerftestSendReportApi {
             return;
         }
 
-        if (!Object.keys(result).length) {
-            console.log('No data to send');
-
-            return;
-        }
-
         const body = this.transform({
             result,
             staticTaskChange,
@@ -52,19 +48,26 @@ class PerftestSendReportApi {
             referrer,
         });
 
-        const response = await got(this.METRICS_URL, {
-            body: JSON.stringify(body),
-            method: 'POST',
-        });
+        const payloads = this.splitRecords(body);
 
-        if (response.statusCode !== 200) {
-            const errorInfo = JSON.parse(response.body);
-            console.log(`Report sent with error: ${errorInfo}`);
+        console.log(`Sending ${payloads.length} chunks`);
 
-            return;
-        }
+        const requests = payloads.map(async (payload, i) => {
+            const response = await got(this.METRICS_URL, {
+                body: JSON.stringify(payload),
+                method: 'POST',
+                retry: { limit: 2, methods: ['POST'] },
+            });
+
+            if (response.statusCode < 200 && response.statusCode > 299) {
+                const errorInfo = JSON.parse(response.body);
+                console.log(`Report chunk #${i} sent with error: ${errorInfo}`);
+            }
+        });
 
-        console.log('Report sent successfully');
+        await Promise.all(requests);
+
+        console.log(`Report sent successfully in ${requests.length} chunks`);
     }
 
     protected transform({
@@ -80,7 +83,7 @@ class PerftestSendReportApi {
         commitHash: string;
         referrer: string;
     }): MetricsRecord[] {
-        const body = [];
+        const body = [] as MetricsRecord[];
         const runId = crypto.randomUUID();
         const data = {
             ...result,
@@ -94,39 +97,145 @@ class PerftestSendReportApi {
         for (const [componentId, subject] of Object.entries(data)) {
             // e.g. taskId = rerender | render
             for (const [taskId, task] of Object.entries(subject)) {
-                const filteredMetrics = Object.entries(task).filter(([k]) => !k.startsWith('_'));
-                // e.g. metricId = median | mean
-                for (const [metricId, metric] of filteredMetrics) {
-                    for (const kind of ['old', 'new', 'change'] as const) {
-                        const metricValue = metric[kind];
-
-                        // change, old are optional
-                        if (!metricValue) {
-                            // eslint-disable-next-line no-continue
-                            continue;
-                        }
-
-                        const item = {
-                            commitHash,
-                            runId,
+                const commonRecordFields = {
+                    commitHash,
+                    runId,
+                    taskId,
+                    referrer,
+                    service,
+                };
+
+                body.push(
+                    ...this.getMetricRecords(task, {
+                        ...commonRecordFields,
+                        componentId,
+                    }),
+                );
+
+                if (Array.isArray(task.modes)) {
+                    const { length } = task.modes;
+
+                    task.modes.forEach((mode, i) => {
+                        const componentVirtualId = `${componentId}${length > 1 ? `_mode${i}` : ''}`;
+                        body.push(
+                            ...this.getMetricRecords(mode, {
+                                ...commonRecordFields,
+                                componentId: componentVirtualId,
+                            }),
+                        );
+                    });
+
+                    body.push(
+                        ...['old', 'new'].map((kind) => ({
+                            ...commonRecordFields,
                             componentId,
-                            taskId,
-                            metricId,
+                            metricId: 'modeCount',
                             kind,
-                            referrer,
-                            service,
-                            payload: this.getPayload(metricValue),
-                        };
-
-                        body.push(item);
+                            payload: this.getPayload(length),
+                        })),
+                    );
+                } else if (task.modes) {
+                    const { new: now, old } = task.modes;
+                    const common = {
+                        ...commonRecordFields,
+                        componentId,
+                        metricId: 'modeCount',
+                    };
+
+                    body.push({
+                        ...common,
+                        kind: 'new',
+                        payload: this.getPayload(now.length),
+                    });
+
+                    if (old) {
+                        body.push({
+                            ...common,
+                            kind: 'old',
+                            payload: this.getPayload(old.length),
+                        });
+
+                        body.push({
+                            ...common,
+                            kind: 'change',
+                            payload: this.getPayload({
+                                difference: now.length - old.length,
+                                percentage: (now.length / (old.length || 1)) * 100,
+                            }),
+                        });
                     }
                 }
             }
         }
 
+        // If there's no result, add the empty mark to see the run on the dashboard
+        if (!body.length) {
+            body.push({
+                commitHash,
+                runId,
+                referrer,
+                service,
+                taskId: 'mark',
+                componentId: 'mark',
+                metricId: 'mark',
+                kind: 'new',
+                payload: this.getPayload(0),
+            });
+        }
+
         return body;
     }
 
+    protected getMetricRecords(
+        task: SubjectResult[string],
+        common: Omit<MetricsRecord, 'kind' | 'payload' | 'metricId'>,
+    ): MetricsRecord[] {
+        const res = [] as MetricsRecord[];
+
+        /* eslint-disable prettier/prettier */
+        const filteredMetrics = Object.entries(task).filter(([k]) => !k.startsWith('_') && !['modes'].includes(k)) as [
+            string,
+            ComparableResult,
+        ][];
+        /* eslint-enable prettier/prettier */
+
+        // e.g. metricId = median | mean
+        for (const [metricId, metric] of filteredMetrics) {
+            for (const kind of ['old', 'new', 'change'] as const) {
+                const metricValue = metric[kind];
+
+                // change, old are optional
+                if (!metricValue) {
+                    // eslint-disable-next-line no-continue
+                    continue;
+                }
+
+                const item = {
+                    ...common,
+                    kind,
+                    metricId,
+                    payload: this.getPayload(metricValue),
+                };
+
+                res.push(item);
+            }
+        }
+
+        return res;
+    }
+
+    protected splitRecords(records: MetricsRecord[]): MetricsRecord[][] {
+        const singleRecordApproxSize = JSON.stringify(records[0]).length;
+        const maxPayloadLength = Math.trunc(this.MAX_BODY_SIZE / singleRecordApproxSize) || 1;
+        const result = [];
+
+        for (let i = 0; i < records.length; i += maxPayloadLength) {
+            result.push(records.slice(i, i + maxPayloadLength));
+        }
+
+        return result;
+    }
+
     protected getPayload(data: Record<string, unknown> | [number, number] | number): string {
         if (typeof data === 'number') {
             data = { 0: data };

actions/perftest-helpers/src/modules/types.ts

@@ -13,8 +13,18 @@ export type ComparableResult = {
     change?: CompareResult;
 };
 
+type Comparable = { [statKey: string]: ComparableResult };
+type SingleResultMap = { [statKey: string]: MetricResult };
+
 export type SubjectResult = {
-    [taskId: string]: { [statKey: string]: ComparableResult };
+    [taskId: string]: Comparable & {
+        modes?:
+            | Comparable[]
+            | {
+                  new: SingleResultMap[];
+                  old?: SingleResultMap[];
+              };
+    };
 };
 
 export type ReportType = {

actions/perftest-helpers/src/modules/writeComment.ts

@@ -62,23 +62,48 @@ ${this.getReportDescription(jsonReport)}
 
     protected getResultByTaskId(jsonReport: ReportType): { [taskId: string]: [string, ComparableResult][] } {
         const resultByTaskId: { [taskId: string]: [string, ComparableResult][] } = {};
+        const modeCountChangeByTaskId: { [taskId: string]: [string, ComparableResult][] } = {};
 
         for (const [subjectId, subjectResult] of Object.entries(jsonReport.result)) {
-            const filteredSubjectResult = Object.entries(subjectResult).filter(
-                ([, taskResult]) => taskResult.mean.change?.significanceRank === 'high',
-            );
+            const filteredSubjectResult = Object.entries(subjectResult).filter(([, taskResult]) => {
+                if (taskResult.modes) {
+                    return true;
+                }
+
+                return taskResult.mean.change?.significanceRank === 'high';
+            });
 
             for (const [taskId, taskResult] of filteredSubjectResult) {
                 resultByTaskId[taskId] = resultByTaskId[taskId] || [];
-                resultByTaskId[taskId].push([subjectId, taskResult.mean]);
+
+                if (Array.isArray(taskResult.modes)) {
+                    taskResult.modes.forEach((mode, i) => {
+                        if (mode.mean.change?.significanceRank === 'high') {
+                            resultByTaskId[taskId].push([`${subjectId}_mode${i + 1}`, mode.mean]);
+                        }
+                    });
+                } else if (taskResult.modes) {
+                    modeCountChangeByTaskId[taskId] = modeCountChangeByTaskId[taskId] || [];
+                    modeCountChangeByTaskId[taskId].push([
+                        `${subjectId}_modeCount`,
+                        { old: taskResult.modes.old?.length, new: taskResult.modes.new.length },
+                    ]);
+                } else {
+                    resultByTaskId[taskId].push([subjectId, taskResult.mean]);
+                }
             }
         }
 
         Object.values(resultByTaskId).forEach((item) =>
             item.sort(([, a], [, b]) => Math.abs(b.change!.percentage) - Math.abs(a.change!.percentage)).splice(5),
         );
 
-        return resultByTaskId;
+        Object.entries(modeCountChangeByTaskId).forEach(([taskId, res]) => {
+            resultByTaskId[taskId] = resultByTaskId[taskId] || [];
+            resultByTaskId[taskId].push(...res);
+        });
+
+        return Object.fromEntries(Object.entries(resultByTaskId).filter(([, res]) => res.length));
     }
 
     protected getDescriptionTables(resultByTaskId: { [taskId: string]: [string, ComparableResult][] }): string {
@@ -95,19 +120,20 @@ ${this.getReportDescription(jsonReport)}
     }
 
     protected getSubjectTaskResultTableRow(subjectId: string, result: ComparableResult): string {
-        const change = this.formatValue(result.change!.difference!, { suffix: ' pts' });
-        const percentage = this.formatValue(result.change!.percentage!, {
-            suffix: '%',
-            sign: true,
-        });
-        const old = this.formatValue(Array.isArray(result.old) ? result.old[0] : result.old!, {
-            suffix: ' pts',
-        });
-        const now = this.formatValue(Array.isArray(result.new) ? result.new[0] : result.new!, {
+        const change = result.change ? this.formatValue(result.change.difference, { suffix: ' pts' }) : '';
+        const percentage = result.change
+            ? ` (${this.formatValue(result.change.percentage, { suffix: '%', sign: true })})`
+            : '';
+        const old = result.old
+            ? this.formatValue(Array.isArray(result.old) ? result.old[0] : result.old, {
+                  suffix: ' pts',
+              })
+            : '';
+        const now = this.formatValue(Array.isArray(result.new) ? result.new[0] : result.new, {
             suffix: ' pts',
         });
 
-        return `| ${subjectId} | ${change} (${percentage}) | ${old} | ${now} |`;
+        return `| ${subjectId} | ${change}${percentage} | ${old} | ${now} |`;
     }
 
     protected formatValue(