Merge branch 'dev' into optimize-metric-query

.jenkins/README.md

@@ -0,0 +1,3 @@
+# Jenkins Documentation
+
+- [Upgrade Frontend and Backend Documentation](https://carnegielearning.atlassian.net/wiki/spaces/RP/pages/3937402881/Jenkins)

README.md

@@ -88,3 +88,6 @@ For an overview of the UpGrade platform, please visit [UpGrade Platform](https:/
 
 ![Built with Science](http://ForTheBadge.com/images/badges/built-with-science.svg)
 
+## Jenkins Documentation
+
+- [Upgrade Frontend and Backend Documentation](https://carnegielearning.atlassian.net/wiki/spaces/RP/pages/3937402881/Jenkins)

backend/packages/Scheduler/package.json

@@ -22,7 +22,7 @@
     "@types/aws-lambda": "^8.10.17",
     "@types/jsonwebtoken": "^8.3.9",
     "@types/node": "^10.17.20",
-    "@types/node-fetch": "^2.5.7",
+    "@types/node-fetch": "^2.6.12",
     "eslint": "^8.27.0",
     "fork-ts-checker-webpack-plugin": "^8.0.0",
     "prettier": "^2.7.1",
@@ -34,4 +34,4 @@
   },
   "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)",
   "license": "MIT"
-}
+}

backend/packages/Scheduler/tsconfig.json

@@ -3,7 +3,7 @@
     "lib": ["es2017", "dom"],
     "removeComments": true,
     "moduleResolution": "node",
-    "noUnusedLocals": true,
+    // "noUnusedLocals": true,
     "noUnusedParameters": true,
     "sourceMap": true,
     "target": "es2017",

backend/packages/Upgrade/src/api/controllers/ExperimentController.ts

@@ -1093,7 +1093,7 @@ export class ExperimentController {
    * @swagger
    * /experiments/{id}:
    *    put:
-   *       description: Create New Experiment
+   *       description: Update Experiment
    *       consumes:
    *         - application/json
    *       parameters:

backend/packages/Upgrade/src/api/controllers/SegmentController.ts

@@ -4,6 +4,7 @@ import { Segment } from '../models/Segment';
 import { AppRequest } from '../../types';
 import {
   IdValidator,
+  ListInputValidator,
   SegmentFile,
   SegmentIds,
   SegmentIdValidator,
@@ -60,6 +61,17 @@ export interface getSegmentData {
  *         items:
  *           type: string
  *           example: '5812a759-1dcf-47a8-b0ba-26c89092863e'
+ *   ListInput:
+ *    allOf:
+ *      - $ref: '#/definitions/Segment'
+ *    required:
+ *      - parentSegmentId
+ *      - listType
+ *    properties:
+ *        parentSegmentId:
+ *         type: string
+ *        listType:
+ *         type: string
  *   segmentResponse:
  *     description: ''
  *     type: object
@@ -343,6 +355,41 @@ export class SegmentController {
     return this.segmentService.upsertSegment(segment, request.logger);
   }
 
+  /**
+   * @swagger
+   * /list:
+   *    post:
+   *      description: Create a new list
+   *      tags:
+   *        - Segment
+   *      produces:
+   *        - application/json
+   *      parameters:
+   *        - in: body
+   *          name: list
+   *          description: List object
+   *          required: true
+   *          schema:
+   *            type: object
+   *            $ref: '#/definitions/ListInput'
+   *      responses:
+   *        '200':
+   *          description: Create a new list
+   *          schema:
+   *            $ref: '#/definitions/segmentResponse'
+   *        '401':
+   *          description: Authorization Required Error
+   *        '500':
+   *          description: Internal Server Error, Insert Error in database, SegmentId is not valid, JSON format is not valid
+   */
+  @Post('/list')
+  public addList(
+    @Body({ validate: true }) listInput: ListInputValidator,
+    @Req() request: AppRequest
+  ): Promise<Segment> {
+    return this.segmentService.addList(listInput, request.logger);
+  }
+
   /**
    * @swagger
    * /segments/{segmentId}:

backend/packages/Upgrade/src/api/controllers/validators/SegmentInputValidator.ts

@@ -24,6 +24,10 @@ export class SegmentInputValidator {
   @IsOptional()
   public description?: string;
 
+  @IsString()
+  @IsOptional()
+  public listType?: string;
+
   @IsNotEmpty()
   @IsString()
   public context: string;
@@ -46,6 +50,12 @@ export class SegmentInputValidator {
   public subSegmentIds: string[];
 }
 
+export class ListInputValidator extends SegmentInputValidator {
+  @IsNotEmpty()
+  @IsUUID()
+  public parentSegmentId: string;
+}
+
 export class IdValidator {
   @IsNotEmpty()
   @IsUUID()

backend/packages/Upgrade/src/api/models/Segment.ts

@@ -22,6 +22,11 @@ export class Segment extends BaseModel {
   })
   public description: string;
 
+  @Column({
+    nullable: true,
+  })
+  public listType: string;
+
   @Column()
   public context: string;
 

backend/packages/Upgrade/src/api/repositories/LogRepository.ts

@@ -137,7 +137,7 @@ export class LogRepository extends Repository<Log> {
       type: IMetricMetaData;
     }>
   > {
-    const experimentRepo = Container.getCustomRepository(ExperimentRepository);
+    const experimentRepo = Container.getCustomRepository(ExperimentRepository, 'export');
     return experimentRepo
       .createQueryBuilder('experiment')
       .select([

backend/packages/Upgrade/src/api/services/AnalyticsService.ts

@@ -30,7 +30,7 @@ import utc from 'dayjs/plugin/utc';
 import timezone from 'dayjs/plugin/timezone';
 import { ExperimentService } from './ExperimentService';
 import { QueryService } from './QueryService';
-import { HttpError } from '../errors';
+import { HttpError } from 'routing-controllers';
 
 dayjs.extend(utc);
 dayjs.extend(timezone);

backend/packages/Upgrade/src/api/services/ExperimentAssignmentService.ts

@@ -29,7 +29,6 @@ import { ExperimentRepository } from '../repositories/ExperimentRepository';
 import { IndividualExclusion } from '../models/IndividualExclusion';
 import { GroupExclusion } from '../models/GroupExclusion';
 import { Experiment } from '../models/Experiment';
-import { ScheduledJobService } from './ScheduledJobService';
 import { ExperimentCondition } from '../models/ExperimentCondition';
 import { v4 as uuid } from 'uuid';
 import { PreviewUserService } from './PreviewUserService';
@@ -51,7 +50,6 @@ import isequal from 'lodash.isequal';
 import flatten from 'lodash.flatten';
 import { ILogInput, ENROLLMENT_CODE } from 'upgrade_types';
 import { StateTimeLogsRepository } from '../repositories/StateTimeLogsRepository';
-import { StateTimeLog } from '../models/StateTimeLogs';
 import { UpgradeLogger } from '../../lib/logger/UpgradeLogger';
 import { SegmentService } from './SegmentService';
 import { MonitoredDecisionPointLogRepository } from '../repositories/MonitoredDecisionPointLogRepository';
@@ -108,7 +106,6 @@ export class ExperimentAssignmentService {
 
     public previewUserService: PreviewUserService,
     public experimentUserService: ExperimentUserService,
-    public scheduledJobService: ScheduledJobService,
     public errorService: ErrorService,
     public settingService: SettingService,
     public segmentService: SegmentService,
@@ -1069,17 +1066,14 @@ export class ExperimentAssignmentService {
   private async checkEnrollmentEndingCriteriaForCount(experiment: Experiment, logger: UpgradeLogger): Promise<void> {
     const { enrollmentCompleteCondition } = experiment;
     const { groupCount, userCount } = enrollmentCompleteCondition;
-
-    const timeLogDate = new Date();
     /**
      * Create stateTimeLog document which will be inserted if ending criteria is met
      */
-    const stateTimeLogDoc = new StateTimeLog();
-    stateTimeLogDoc.id = uuid();
-    stateTimeLogDoc.fromState = experiment.state;
-    stateTimeLogDoc.toState = EXPERIMENT_STATE.ENROLLMENT_COMPLETE;
-    stateTimeLogDoc.timeLog = timeLogDate;
-    stateTimeLogDoc.experiment = experiment;
+    const stateTimeLogDoc = await this.experimentService.prepareStateTimeLogDoc(
+      experiment,
+      experiment.state,
+      EXPERIMENT_STATE.ENROLLMENT_COMPLETE
+    );
 
     if (groupCount && userCount && experiment.assignmentUnit === ASSIGNMENT_UNIT.GROUP) {
       const groupSatisfied: number = await this.getGroupAssignmentStatus(experiment.id, logger);

backend/packages/Upgrade/src/api/services/ExperimentService.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-var */
 import { GroupExclusion } from './../models/GroupExclusion';
 import { ErrorWithType } from './../errors/ErrorWithType';
-import { Service } from 'typedi';
+import { Inject, Service } from 'typedi';
 import { InjectDataSource, InjectRepository } from '../../typeorm-typedi-extensions';
 import { ExperimentRepository } from '../repositories/ExperimentRepository';
 import {
@@ -123,6 +123,7 @@ export class ExperimentService {
     @InjectDataSource() private dataSource: DataSource,
     public previewUserService: PreviewUserService,
     public segmentService: SegmentService,
+    @Inject(() => ScheduledJobService)
     public scheduledJobService: ScheduledJobService,
     public errorService: ErrorService,
     public cacheService: CacheService,
@@ -402,6 +403,20 @@ export class ExperimentService {
     return [...conditionIds, ...decisionPointsIds];
   }
 
+  public async prepareStateTimeLogDoc(
+    experimentDoc: Experiment,
+    fromState: EXPERIMENT_STATE,
+    toState: EXPERIMENT_STATE
+  ) {
+    const stateTimeLogDoc = new StateTimeLog();
+    stateTimeLogDoc.id = uuid();
+    stateTimeLogDoc.fromState = fromState;
+    stateTimeLogDoc.toState = toState;
+    stateTimeLogDoc.timeLog = new Date();
+    stateTimeLogDoc.experiment = experimentDoc;
+    return stateTimeLogDoc;
+  }
+
   public async updateState(
     experimentId: string,
     state: EXPERIMENT_STATE,
@@ -458,14 +473,7 @@ export class ExperimentService {
     // add experiment audit logs
     await this.experimentAuditLogRepository.saveRawJson(LOG_TYPE.EXPERIMENT_STATE_CHANGED, data, user, entityManager);
 
-    const timeLogDate = new Date();
-
-    const stateTimeLogDoc = new StateTimeLog();
-    stateTimeLogDoc.id = uuid();
-    stateTimeLogDoc.fromState = oldExperiment.state;
-    stateTimeLogDoc.toState = state;
-    stateTimeLogDoc.timeLog = timeLogDate;
-    stateTimeLogDoc.experiment = oldExperiment;
+    const stateTimeLogDoc = await this.prepareStateTimeLogDoc(oldExperiment, oldExperiment.state, state);
 
     // updating the experiment and stateTimeLog
     const stateTimeLogRepo = entityManager ? entityManager.getRepository(StateTimeLog) : this.stateTimeLogsRepository;
@@ -784,6 +792,13 @@ export class ExperimentService {
         let experimentDoc: Experiment;
         try {
           experimentDoc = await transactionalEntityManager.getRepository(Experiment).save(expDoc);
+          // Store state time log for the experiment
+          const stateTimeLogDoc = await this.prepareStateTimeLogDoc(
+            experimentDoc,
+            oldExperiment.state,
+            experimentDoc.state
+          );
+          await transactionalEntityManager.getRepository(StateTimeLog).save(stateTimeLogDoc);
         } catch (err) {
           const error = err as ErrorWithType;
           error.details = `Error in updating experiment document "updateExperimentInDB"`;
@@ -1227,6 +1242,15 @@ export class ExperimentService {
       let experimentDoc: Experiment;
       try {
         experimentDoc = await transactionalEntityManager.getRepository(Experiment).save(expDoc);
+        // Store state time log for the experiment in enrolling state.
+        if (experimentDoc.state !== EXPERIMENT_STATE.INACTIVE) {
+          const stateTimeLogDoc = await this.prepareStateTimeLogDoc(
+            experimentDoc,
+            EXPERIMENT_STATE.INACTIVE,
+            experimentDoc.state
+          );
+          await transactionalEntityManager.getRepository(StateTimeLog).save(stateTimeLogDoc);
+        }
       } catch (err) {
         const error = err as ErrorWithType;
         error.details = 'Error in adding experiment in DB';

backend/packages/Upgrade/src/api/services/ScheduledJobService.ts

@@ -44,7 +44,7 @@ export class ScheduledJobService {
         if (scheduledJob && scheduledJob.experiment) {
           const experimentRepository = transactionalEntityManager.getRepository(Experiment);
           const experiment = await experimentRepository.findOneBy({ id: scheduledJob.experiment.id });
-          if (scheduledJob && experiment) {
+          if (experiment) {
             const systemUser = await transactionalEntityManager
               .getRepository(User)
               .findOneBy({ email: systemUserDoc.email });

backend/packages/Upgrade/src/api/services/SegmentService.ts

@@ -26,6 +26,7 @@ import {
   SegmentFile,
   Group,
   SegmentValidationObj,
+  ListInputValidator,
 } from '../controllers/validators/SegmentInputValidator';
 import { ExperimentSegmentExclusionRepository } from '../repositories/ExperimentSegmentExclusionRepository';
 import { ExperimentSegmentInclusionRepository } from '../repositories/ExperimentSegmentInclusionRepository';
@@ -297,6 +298,25 @@ export class SegmentService {
     return this.addSegmentDataInDB(segment, logger);
   }
 
+  public async addList(listInput: ListInputValidator, logger: UpgradeLogger): Promise<Segment> {
+    logger.info({ message: `Adding list => ${JSON.stringify(listInput, undefined, 2)}` });
+    const manager = this.dataSource;
+    const { parentSegmentId, ...segmentInput } = listInput;
+    const newList: SegmentInputValidator = { ...segmentInput, type: SEGMENT_TYPE.PRIVATE };
+    const createdSegment = await manager.transaction(async (transactionalEntityManager) => {
+      const createdSegment = await this.upsertSegmentInPipeline(newList, logger, transactionalEntityManager);
+      const parentSegment = await this.getSegmentById(parentSegmentId, logger);
+      if (!parentSegment) {
+        throw new Error('Parent Segment not found');
+      }
+      parentSegment.subSegments = [...parentSegment.subSegments, createdSegment];
+
+      await transactionalEntityManager.getRepository(Segment).save(parentSegment);
+      return createdSegment;
+    });
+    return createdSegment;
+  }
+
   public upsertSegmentInPipeline(
     segment: SegmentInputValidator,
     logger: UpgradeLogger,
@@ -308,7 +328,34 @@ export class SegmentService {
 
   public async deleteSegment(id: string, logger: UpgradeLogger): Promise<Segment> {
     logger.info({ message: `Delete segment by id. segmentId: ${id}` });
-    return await this.segmentRepository.deleteSegment(id, logger);
+    const manager = this.dataSource;
+    const deletedSegment = manager.transaction(async (transactionalEntityManager) => {
+      return this.deleteSegmentAndPrivateSubsegments(id, logger, transactionalEntityManager);
+    });
+    return deletedSegment;
+  }
+
+  private async deleteSegmentAndPrivateSubsegments(
+    id: string,
+    logger: UpgradeLogger,
+    manager: EntityManager
+  ): Promise<Segment> {
+    const segmentDoc = await manager.getRepository(Segment).findOne({
+      where: { id: id },
+      relations: ['individualForSegment', 'groupForSegment', 'subSegments'],
+    });
+    if (!segmentDoc) {
+      throw new Error(SERVER_ERROR.QUERY_FAILED);
+    }
+    await Promise.all(
+      segmentDoc.subSegments.map((subSegment) => {
+        if (subSegment.type === SEGMENT_TYPE.PRIVATE) {
+          this.deleteSegmentAndPrivateSubsegments(subSegment.id, logger, manager);
+        }
+      })
+    );
+    const deletedSegmentResponse = await this.segmentRepository.deleteSegments([id], logger, manager);
+    return deletedSegmentResponse[0];
   }
 
   public async validateSegments(segments: SegmentFile[], logger: UpgradeLogger): Promise<SegmentImportError[]> {
@@ -584,7 +631,7 @@ export class SegmentService {
 
     // create/update segment document
     segment.id = segment.id || uuid();
-    const { id, name, description, context, type } = segment;
+    const { id, name, description, context, type, listType } = segment;
     const allSegments = await this.getSegmentByIds(segment.subSegmentIds);
     const subSegmentData = segment.subSegmentIds
       .filter((subSegmentId) => {
@@ -610,6 +657,7 @@ export class SegmentService {
         description,
         context,
         type,
+        listType,
         subSegments: subSegmentData,
       });
     } catch (err) {