Skip to content

Commit

Permalink
feat: improve export (#316)
Browse files Browse the repository at this point in the history 
* feat(export): add support elasticsearch lang
* feat: improve handle error
* feat(export): limit export to 10k documents
  prevent segfault with more than 25k results
* fix(export): prevent error duplicate result
  • Loading branch information
@sebtiz13
sebtiz13 authored Aug 31, 2023
1 parent ab01714 commit e4e746e
Show file tree
Hide file tree
Showing 13 changed files with 577 additions and 336 deletions.
92 changes: 65 additions & 27 deletions lib/modules/asset/AssetsController.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { ControllerDefinition, HttpStream, KuzzleRequest } from "kuzzle";
import {
ControllerDefinition,
HttpStream,
KuzzleError,
KuzzleRequest,
} from "kuzzle";

import { MeasureExporter } from "../measure/";
import { DeviceManagerPlugin, InternalCollection } from "../plugin";
Expand Down Expand Up @@ -200,12 +205,14 @@ export class AssetsController {
const query = request.input.body?.query;
const sort = request.input.body?.sort;
const type = request.input.args.type;
const lang = request.getLangParam();

const { measures, total } = await this.measureExporter.search(
engineId,
{
endAt,
id,
lang,
query,
sort,
startAt,
Expand All @@ -227,20 +234,35 @@ export class AssetsController {
request.context.connection.protocol === "http" &&
request.context.connection.misc.verb === "GET"
) {
const exportId = request.getString("exportId");

const { id } = await this.measureExporter.getExport(engineId, exportId);

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="asset-${id}.csv"`,
"Content-Type": "text/csv",
},
});

const stream = await this.measureExporter.sendExport(engineId, exportId);

return new HttpStream(stream);
try {
const exportId = request.getString("exportId");

const { id } = await this.measureExporter.getExport(engineId, exportId);
const stream = await this.measureExporter.sendExport(
engineId,
exportId
);

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="asset-${id}.csv"`,
"Content-Type": "text/csv",
},
});

return new HttpStream(stream);
} catch (error) {
// ? Like this endpoint is mostly called by browser prefer return raw message for essyier readable error
request.response.configure({
format: "raw",
headers: {
"Content-Type": "text/plain",
},
status: (error as KuzzleError).status,
});

return error.message;
}
}

const id = request.getId();
Expand All @@ -251,13 +273,15 @@ export class AssetsController {
const query = request.input.body?.query;
const sort = request.input.body?.sort;
const type = request.input.args.type;
const lang = request.getLangParam();

const link = await this.measureExporter.prepareExport(
engineId,
request.getUser(),
{
endAt,
id,
lang,
query,
sort,
startAt,
Expand All @@ -275,27 +299,41 @@ export class AssetsController {
request.context.connection.protocol === "http" &&
request.context.connection.misc.verb === "GET"
) {
const exportId = request.getString("exportId");

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="${InternalCollection.ASSETS}.csv"`,
"Content-Type": "text/csv",
},
});

const stream = await this.exporter.sendExport(engineId, exportId);

return new HttpStream(stream);
try {
const exportId = request.getString("exportId");
const stream = await this.exporter.sendExport(engineId, exportId);

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="${InternalCollection.ASSETS}.csv"`,
"Content-Type": "text/csv",
},
});

return new HttpStream(stream);
} catch (error) {
// ? Like this endpoint is mostly called by browser prefer return raw message for essyier readable error
request.response.configure({
format: "raw",
headers: {
"Content-Type": "text/plain",
},
status: (error as KuzzleError).status,
});

return error.message;
}
}

const query = request.input.body?.query;
const sort = request.input.body?.sort;
const lang = request.getLangParam();

const link = await this.exporter.prepareExport(
engineId,
request.getUser(),
{
lang,
query,
sort,
}
Expand Down
6 changes: 6 additions & 0 deletions lib/modules/asset/types/AssetApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ export interface ApiAssetGetMeasuresRequest extends AssetsControllerRequest {

type?: string;

lang?: "koncorde" | "elasticsearch";

body?: {
query?: JSONObject;
sort?: JSONObject;
Expand Down Expand Up @@ -114,6 +116,8 @@ export interface ApiAssetExportMeasuresRequest extends AssetsControllerRequest {

type?: string;

lang?: "koncorde" | "elasticsearch";

body?: {
query?: JSONObject;
sort?: JSONObject;
Expand All @@ -126,6 +130,8 @@ export type ApiAssetExportMeasuresResult = {
export interface ApiAssetExportRequest extends AssetsControllerRequest {
action: "export";

lang?: "koncorde" | "elasticsearch";

body?: {
query?: JSONObject;
sort?: JSONObject;
Expand Down
82 changes: 58 additions & 24 deletions lib/modules/device/DevicesController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
BadRequestError,
ControllerDefinition,
HttpStream,
KuzzleError,
KuzzleRequest,
} from "kuzzle";
import _ from "lodash";
Expand Down Expand Up @@ -355,12 +356,14 @@ export class DevicesController {
const query = request.input.body?.query;
const sort = request.input.body?.sort;
const type = request.input.args.type;
const lang = request.getLangParam();

const { measures, total } = await this.measureExporter.search(
engineId,
{
endAt,
id,
lang,
query,
sort,
startAt,
Expand All @@ -379,20 +382,35 @@ export class DevicesController {
request.context.connection.protocol === "http" &&
request.context.connection.misc.verb === "GET"
) {
const exportId = request.getString("exportId");

const { id } = await this.measureExporter.getExport(engineId, exportId);

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="device-${id}.csv"`,
"Content-Type": "text/csv",
},
});
try {
const exportId = request.getString("exportId");

const stream = await this.measureExporter.sendExport(engineId, exportId);
const { id } = await this.measureExporter.getExport(engineId, exportId);
const stream = await this.measureExporter.sendExport(
engineId,
exportId
);

return new HttpStream(stream);
request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="device-${id}.csv"`,
"Content-Type": "text/csv",
},
});

return new HttpStream(stream);
} catch (error) {
// ? Like this endpoint is mostly called by browser prefer return raw message for essyier readable error
request.response.configure({
format: "raw",
headers: {
"Content-Type": "text/plain",
},
status: (error as KuzzleError).status,
});

return error.message;
}
}

const id = request.getId();
Expand All @@ -403,13 +421,15 @@ export class DevicesController {
const query = request.input.body?.query;
const sort = request.input.body?.sort;
const type = request.input.args.type;
const lang = request.getLangParam();

const link = await this.measureExporter.prepareExport(
engineId,
request.getUser(),
{
endAt,
id,
lang,
query,
sort,
startAt,
Expand Down Expand Up @@ -471,27 +491,41 @@ export class DevicesController {
request.context.connection.protocol === "http" &&
request.context.connection.misc.verb === "GET"
) {
const exportId = request.getString("exportId");

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="${InternalCollection.DEVICES}.csv"`,
"Content-Type": "text/csv",
},
});

const stream = await this.exporter.sendExport(engineId, exportId);

return new HttpStream(stream);
try {
const exportId = request.getString("exportId");
const stream = await this.exporter.sendExport(engineId, exportId);

request.response.configure({
headers: {
"Content-Disposition": `attachment; filename="${InternalCollection.DEVICES}.csv"`,
"Content-Type": "text/csv",
},
});

return new HttpStream(stream);
} catch (error) {
// ? Like this endpoint is mostly called by browser prefer return raw message for essyier readable error
request.response.configure({
format: "raw",
headers: {
"Content-Type": "text/plain",
},
status: (error as KuzzleError).status,
});

return error.message;
}
}

const query = request.input.body?.query;
const sort = request.input.body?.sort;
const lang = request.getLangParam();

const link = await this.exporter.prepareExport(
engineId,
request.getUser(),
{
lang,
query,
sort,
}
Expand Down
6 changes: 6 additions & 0 deletions lib/modules/device/types/DeviceApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ export interface ApiDeviceGetMeasuresRequest extends DevicesControllerRequest {

type?: string;

lang?: "koncorde" | "elasticsearch";

body?: {
query?: JSONObject;
sort?: JSONObject;
Expand Down Expand Up @@ -195,6 +197,8 @@ export interface ApiDeviceExportMeasuresRequest

type?: string;

lang?: "koncorde" | "elasticsearch";

body?: {
query?: JSONObject;
sort?: JSONObject;
Expand Down Expand Up @@ -226,6 +230,8 @@ export type ApiDeviceReceiveMeasuresResult = void;
export interface ApiDeviceExportRequest extends DevicesControllerRequest {
action: "export";

lang?: "koncorde" | "elasticsearch";

body?: {
query?: JSONObject;
sort?: JSONObject;
Expand Down
Loading

0 comments on commit e4e746e

Please sign in to comment.