Skip to content

Commit

Permalink
Updated stratification controller APIs
Browse files Browse the repository at this point in the history
  • Loading branch information
RidhamShah committed Sep 19, 2023
1 parent eb9571a commit ef99a0b
Show file tree
Hide file tree
Showing 2 changed files with 198 additions and 38 deletions.
137 changes: 117 additions & 20 deletions backend/packages/Upgrade/src/api/controllers/StratificationController.ts
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,32 @@
import { JsonController, Get, Delete, Param, Authorized, Post, Req, Body } from 'routing-controllers';
import { JsonController, Get, Delete, Param, Authorized, Post, Req, UseBefore, Res, Body, UploadedFile } from 'routing-controllers';
import { SERVER_ERROR } from 'upgrade_types';
import { isUUID } from 'class-validator';
import { AppRequest } from '../../types';
import { UserStratificationFactor } from '../models/UserStratificationFactor';
import { StratificationService } from '../services/StratificationService';
import { FactorStrata, StratificationInputValidator } from './validators/StratificationValidator';
import { StratificationFactor } from '../models/StratificationFactor';
import * as express from 'express';
import { Parser } from 'json2csv';

// const fs = require("fs");
// const { parse } = require("csv-parse");
import formidable, { errors as formidableErrors } from 'formidable';
import { CSVMiddleware } from '../middlewares/BodyParserMiddleware copy';
import multer from 'multer';
const storage = multer.memoryStorage();
const upload = multer({ storage: storage });
const fs = require('fs');
const { parse } = require('csv-parse');


// import { CSVMiddleware } from '../middlewares/BodyParserMiddleware copy';
// import multer from 'multer';
import { createObjectCsvWriter } from 'csv-writer';

// //const multer = require('multer');
// let storage = multer.memoryStorage();
// let upload = multer({ storage: storage });
// const fs = require('fs');
// const { parse } = require('csv-parse');

/**
* @swagger
Expand Down Expand Up @@ -180,7 +198,7 @@ import { StratificationFactor } from '../models/StratificationFactor';
* @swagger
* tags:
* - name: Stratification
* description: CRUD operations related to Segment
* description: CRUD operations related to Stratification
*/
@Authorized()
@JsonController('/stratification')
Expand Down Expand Up @@ -240,10 +258,12 @@ export class StratificationController {
* description: Internal Server Error, SegmentId is not valid
*/
@Get('/download/:factor')
public getStratificationByFactorId(
// @UseBefore(CSVMiddleware)
public async getStratificationByFactorId(
@Param('factor') factor: string,
@Req() request: AppRequest
): Promise<FactorStrata> {
@Req() request: AppRequest,
@Res() res: express.Response
): Promise<any> {
if (!factor) {
return Promise.reject(new Error(SERVER_ERROR.MISSING_PARAMS + ' : stratification Factor should not be null.'));
}
Expand All @@ -257,7 +277,58 @@ export class StratificationController {
)
);
}
return this.stratificatonService.getStratificationByFactor(factor, request.logger);

// const csv = JSONToCSV(req.body, { fields: ["Customer Name", "Business Name", "Customer Email", "Customer ID", "School ID", "Student Count", "Admin Count", "Courses Count" ]})

// res.attachment('customers.csv').send(csv)

/*const data = [
{ name: 'Alice', age: 25, city: 'New York' },
{ name: 'Bob', age: 30, city: 'Los Angeles' },
{ name: 'Charlie', age: 22, city: 'Chicago' },
];
// Create a CSV writer
const csvWriter = createObjectCsvWriter({
path: 'data.csv',
header: [
{ id: 'name', title: 'Name' },
{ id: 'age', title: 'Age' },
{ id: 'city', title: 'City' },
],
});
// Write the data to the CSV file
csvWriter
.writeRecords(data)
.then(() => {
// Set the response headers to specify a CSV content type and trigger download
res.setHeader('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', 'attachment; filename=data.csv');
res.set('Content-Type', 'text/csv; charset=UTF-8');
// Pipe the CSV file to the response
const fileStream = fs.createReadStream('data.csv');
fileStream.pipe(res);
})
.catch((error) => {
console.error('Error writing CSV:', error);
res.status(500).send('Internal Server Error');
});*/

const data = await this.stratificatonService.getCSVDataByFactor(factor, request.logger);
const parser = new Parser();
// Convert JSON data to CSV
const csv = parser.parse(data);

// return csv file with appropriate headers to request;


console.log(csv);
res.setHeader('Content-Type', 'text/csv; charset=UTF-8');
res.setHeader('Content-Disposition', 'attachment; filename="data.csv"');
res.send(csv);
return;
}

/**
Expand Down Expand Up @@ -288,18 +359,44 @@ export class StratificationController {
* description: Internal Server Error, Insert Error in database, SegmentId is not valid, JSON format is not valid
*/
@Post()
public insertStratification(
@Body({ validate: false }) temp: StratificationInputValidator[],
@Req() request: AppRequest
): Promise<UserStratificationFactor[]> {
// read csv file
// const { file } = request;
// fs.createReadStream(file)
// .pipe(parse({ delimiter: ",", from_line: 2 }))
// .on("data", function (row) {
// console.log(row);
// })
return this.stratificatonService.insertStratification(temp, request.logger);
@UseBefore(upload.single('file'))
public insertStratification(@Req() request: AppRequest): Promise<UserStratificationFactor[]> {
const csvData = request.file['buffer'].toString();
console.log(csvData.toString()); // TODO: remove this

const rows = csvData.split('\n');
const columnNames = rows[0].split(',');

const userFactorValues: StratificationInputValidator[] = [];

// Iterate through the rows (skip the header row)
for (let i = 1; i < rows.length; i++) {
const row = rows[i];
const rowValues = row.split(',');

// Extract the user ID
const userId = rowValues[0];
if (!userId) {
continue;
}

// Iterate through other columns (factors)
for (let j = 1; j < columnNames.length; j++) {
const factorName = columnNames[j];
const factorValue = rowValues[j];

// Create an object and add it to the array
const userFactorValue: StratificationInputValidator = {
userId: userId,
factor: factorName.trim(),
value: factorValue ? factorValue.trim() : null,
};

userFactorValues.push(userFactorValue);
}
}

return this.stratificatonService.insertStratification(userFactorValues, request.logger);
}

/**
Expand Down
99 changes: 81 additions & 18 deletions backend/packages/Upgrade/src/api/services/StratificationService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { ExperimentUser } from '../models/ExperimentUser';
import { StratificationFactor } from '../models/StratificationFactor';
import { UserStratificationFactor } from '../models/UserStratificationFactor';
import { StratificationFactorRepository } from '../repositories/StratificationFactorRepository';
import uuid from 'uuid';
import { ErrorWithType } from '../errors/ErrorWithType';
@Service()
export class StratificationService {
constructor(
Expand Down Expand Up @@ -72,6 +74,18 @@ export class StratificationService {
return this.calculateStratificationResult(queryBuilder)[0];
}

public async getCSVDataByFactor(factor: string, logger: UpgradeLogger): Promise<any> {
logger.info({ message: `Download CSV stratification by factor. factorId: ${factor}` });

return await this.userStratificationRepository
.createQueryBuilder('usf')
.select(['user.id AS user', 'usf.stratificationFactorValue AS value'])
.innerJoin('usf.user', 'user')
.innerJoin('usf.stratificationFactor', 'sf')
.where('sf.id = :factor', { factor })
.getRawMany();
}

public async deleteStratification(factor: string, logger: UpgradeLogger): Promise<StratificationFactor> {
logger.info({ message: `Delete stratification by factor. factorId: ${factor}` });

Expand All @@ -88,32 +102,81 @@ export class StratificationService {
const userDetails = await transactionalEntityManager.getRepository(ExperimentUser).find({
where: { id: In(userStratificationData.map((userData) => userData.userId)) },
});
const usersRemaining = userStratificationData
.filter((userData) => {
return !userDetails.some((user) => user.id === userData.userId);
})
.map((x) => {
return {
id: x.userId,
};
});
const usersDocToSave = [...new Set(usersRemaining)];

const stratificationFactorDetials = await transactionalEntityManager.getRepository(StratificationFactor).find({
where: { stratificationFactorName: In(userStratificationData.map((factorData) => factorData.factor)) },
});
const stratificationFactorRemaining = userStratificationData.filter((factorData) => {
return !stratificationFactorDetials.some((factor) => factor.id === factorData.factor);
});
const stratificationFactorToSave = [...new Set(stratificationFactorRemaining)].map((x) => {
return { id: uuid(), stratificationFactorName: x.factor };
});

let userDocCreated: ExperimentUser[], stratificationFactorDocCreated: StratificationFactor[];
try {
[userDocCreated, stratificationFactorDocCreated] = await Promise.all([
transactionalEntityManager.getRepository(ExperimentUser).save(usersDocToSave),
transactionalEntityManager.getRepository(StratificationFactor).save(stratificationFactorToSave),
]);
} catch (err) {
const error = err as ErrorWithType;
error.details = 'Error in creating not-founded Experiment User & Stratification factors';
error.type = SERVER_ERROR.QUERY_FAILED;
logger.error(error);
throw error;
}

userDetails.push(...userDocCreated);
stratificationFactorDetials.push(...stratificationFactorDocCreated);

const userStratificationDataToSave = userStratificationData.map((data) => {
const userFound = userDetails.find((user) => user.id === data.userId);
const stratificationFactorFound = stratificationFactorDetials.find(
const userStratificationDataToSave: Partial<UserStratificationFactor>[] = userStratificationData.map((data) => {
let userFound: ExperimentUser = userDetails.find((user) => user.id === data.userId);
let stratificationFactorFound: StratificationFactor = stratificationFactorDetials.find(
(factor) => factor.stratificationFactorName === data.factor
);

if (!userFound) {
const error = new Error('User: ' + data.userId + ' not found.');
(error as any).type = SERVER_ERROR.QUERY_FAILED;
logger.error(error);
} else if (!stratificationFactorFound) {
const error = new Error('StratificationFactor: ' + data.factor + ' not found. ');
(error as any).type = SERVER_ERROR.QUERY_FAILED;
logger.error(error);
} else {
return {
user: userFound,
stratificationFactor: stratificationFactorFound,
stratificationFactorValue: data.value,
};
}
// if (!userFound) {
// try {
// userFound = await transactionalEntityManager.getRepository(ExperimentUser).save({ id: data.userId });
// userDetails.push(userFound);
// } catch (err) {
// const error = err as ErrorWithType;
// error.details = 'Error in creating not-founded Experiment User';
// error.type = SERVER_ERROR.QUERY_FAILED;
// logger.error(error);
// throw error;
// }
// }
// if (!stratificationFactorFound) {
// try {
// stratificationFactorFound = await transactionalEntityManager
// .getRepository(StratificationFactor)
// .save({ id: uuid(), stratificationFactorName: data.factor });
// stratificationFactorDetials.push(stratificationFactorFound);
// } catch (err) {
// const error = err as ErrorWithType;
// error.details = 'Error in creating not-founded Stratification Factor';
// error.type = SERVER_ERROR.QUERY_FAILED;
// logger.error(error);
// throw error;
// }
// }
return {
user: userFound,
stratificationFactor: stratificationFactorFound,
stratificationFactorValue: data.value,
};
});

return await transactionalEntityManager
Expand Down

0 comments on commit ef99a0b

Please sign in to comment.