Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[api] improve insert performance when persisting features data from CSV uploads [MRXN23-256] #1413

Merged
merged 2 commits into from
Aug 1, 2023
Merged

[api] improve insert performance when persisting features data from CSV uploads [MRXN23-256] #1413

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3423576
Add logs and cte optimization to csv features upload
yulia-bel Jul 31, 2023
bc6ac63
Update logs for csv features upload
yulia-bel Aug 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 55 additions & 6 deletions api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ export class FeatureAmountUploadService {

let newFeaturesFromCsvUpload;
try {
this.logger.log(
`Starting process of parsing csv file and saving amount data to temporary storage`,
);
await this.events.createEvent(data);
// saving feature data to temporary table
const featuresRegistry = await this.saveCsvToRegistry(
Expand All @@ -63,13 +66,17 @@ export class FeatureAmountUploadService {
}
// Saving new features to apiDB 'features' table

this.logger.log(`Saving new features to (apiBD).features table...`);
newFeaturesFromCsvUpload = await this.saveNewFeaturesFromCsvUpload(
apiQueryRunner,
featuresRegistry.right.id,
data.projectId,
);

this.logger.log(`New features saved in (apiBD).features table`);
// Saving new features amounts and geoms to geoDB 'features_amounts' table
this.logger.log(
`Starting the process of saving new features amounts and geoms to (geoDB).features_amounts table`,
);
await this.saveNewFeaturesAmountsFromCsvUpload(
newFeaturesFromCsvUpload,
apiQueryRunner,
Expand All @@ -79,18 +86,25 @@ export class FeatureAmountUploadService {
);

// Removing temporary data from apiDB uploads tables
this.logger.log(
`Removing data from temporary tables after successful upload...`,
);
await apiQueryRunner.manager.delete(FeatureAmountUploadRegistry, {
id: featuresRegistry.right.id,
});

this.logger.log(
`Upload temporary data removed from apiDB uploads tables`,
);
// Setting project source to legacy-import to create puvspr.dat files from pre-calculated amounts, to allow to use new features after upload

this.logger.log(`Updating project sources value to legacy-import...`);
await this.updateProjectSources(
data.projectId,
ProjectSourcesEnum.legacyImport,
apiQueryRunner.manager,
);

this.logger.log(`Csv file upload process finished successfully`);
// Committing transaction

await apiQueryRunner.commitTransaction();
Expand Down Expand Up @@ -122,9 +136,14 @@ export class FeatureAmountUploadService {
queryRunner: QueryRunner,
): Promise<any> {
try {
this.logger.log(`Parsing csv file...`);

const parsedFile = await featureAmountCsvParser(data.fileBuffer);

const { featureNames, puids } = this.getFeatureNamesAndPuids(parsedFile);

this.logger.log(`Validating parsed csv file...`);

if (
await this.areFeatureNamesNotAlreadyUsedInProject(
data.projectId,
Expand All @@ -138,6 +157,7 @@ export class FeatureAmountUploadService {
return left(unknownPuidsInFeatureAmountCsvUpload);
}

this.logger.log(`Saving parsed data to temporary storage...`);
const importedRegistry = await this.saveFeaturesToRegistry(
parsedFile,
data.projectId,
Expand All @@ -163,20 +183,24 @@ export class FeatureAmountUploadService {
features,
CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS,
);

this.logger.log(`Saving a new upload data to temporary table...`);
const newUpload = await entityManager
.getRepository(FeatureAmountUploadRegistry)
.save({
projectId,
userId,
});
for (const chunk of featuresChunks) {
for (const [index, chunk] of featuresChunks.entries()) {
this.logger.log(`Inserting chunk ${index} to temporary table...`);
await entityManager
.createQueryBuilder()
.insert()
.into(UploadedFeatureAmount)
.values(chunk.map((feature) => ({ ...feature, upload: newUpload })))
.execute();
}
this.logger.log(`New csv upload data from saved to temporary tables`);
return newUpload;
}

Expand Down Expand Up @@ -219,7 +243,12 @@ export class FeatureAmountUploadService {
uploadId: string,
projectId: string,
) {
for (const newFeature of newFeaturesFromCsvUpload) {
for (const [index, newFeature] of newFeaturesFromCsvUpload.entries()) {
this.logger.log(
`Getting feature amounts for feature number ${index + 1}: ${
newFeature.feature_class_name
}`,
);
const featureAmounts = await apiQueryRunner.manager
.createQueryBuilder()
.select(['fa.puid', 'fa.amount'])
Expand All @@ -235,9 +264,18 @@ export class FeatureAmountUploadService {
CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS,
);

for (const featureChunk of featuresChunks) {
this.logger.log(
`Feature data divided into ${featuresChunks.length} chunks`,
);
for (const [amountIndex, featureChunk] of featuresChunks.entries()) {
this.logger.log(
`Starting the process of saving chunk with index ${amountIndex} of amounts of feature ${newFeature.feature_class_name}...`,
);
const firstParameterNumber = 2;
const parameters: any[] = [projectId];
this.logger.log(
`Generating values to insert for chunk with index ${amountIndex}...`,
);
const valuesToInsert = featureChunk.map((featureAmount, index) => {
parameters.push(
...[
Expand All @@ -260,9 +298,13 @@ export class FeatureAmountUploadService {
)
`;
});

this.logger.log(
`Inserting amount values of chunk with index ${amountIndex} into (geoDB).features_data table...`,
);
await geoQueryRunner.manager.query(
`
WITH project_pus AS (
WITH project_pus AS NOT MATERIALIZED (
SELECT ppu.id, ppu.puid, pug.the_geom FROM projects_pu ppu JOIN planning_units_geom pug ON pug.id = ppu.geom_id WHERE ppu.project_id = $1
)
INSERT INTO features_data (the_geom, feature_id, amount, project_pu_id)
Expand All @@ -272,8 +314,15 @@ export class FeatureAmountUploadService {
`,
parameters,
);
this.logger.log(
`Chunk with index ${amountIndex} saved to (geoDB).features_data`,
);
}
this.logger.log(
`All chunks of feature ${newFeature.feature_class_name} saved`,
);
}
this.logger.log(`All new features data saved to (geoDB).features_data`);
}

private async areFeatureNamesNotAlreadyUsedInProject(
Expand Down