From 0b0c52134b2252f33df5c1d82bf2b1d08ab8df6b Mon Sep 17 00:00:00 2001 From: ssettipalli Date: Tue, 28 May 2024 12:01:18 -0500 Subject: [PATCH] JN-1055 import kitrequest data (#911) Co-authored-by: Sampath K. Settipalli --- .../pearl/core/dao/dataimport/ImportDao.java | 3 + .../core/dao/dataimport/ImportItemDao.java | 4 + .../service/dataimport/ImportItemService.java | 3 + .../service/dataimport/ImportService.java | 7 + .../service/export/EnrolleeImportService.java | 47 +++++- .../item/KitRequestTypeFormatter.java | 17 --- .../formatters/item/KitTypeFormatter.java | 33 ++++ .../module/KitRequestFormatter.java | 44 +++++- .../core/service/kit/KitRequestService.java | 51 ++++--- .../study/StudyEnvironmentService.java | 7 +- .../export/EnrolleeImportServiceTests.java | 143 +++++++++++++++--- 11 files changed, 295 insertions(+), 64 deletions(-) delete mode 100644 core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitRequestTypeFormatter.java create mode 100644 core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitTypeFormatter.java diff --git a/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportDao.java b/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportDao.java index 652cde513a..22de466648 100644 --- a/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportDao.java +++ b/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportDao.java @@ -26,5 +26,8 @@ public List findAllByStudyEnv(UUID studyEnvId) { return findAllByProperty("study_environment_id", studyEnvId); } + public void deleteByStudyEnvId(UUID studyEnvId) { + deleteByProperty("study_environment_id", studyEnvId); + } } diff --git a/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportItemDao.java b/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportItemDao.java index af600137f1..452833e284 100644 --- a/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportItemDao.java +++ b/core/src/main/java/bio/terra/pearl/core/dao/dataimport/ImportItemDao.java @@ -24,4 +24,8 @@ public List findAllByImport(UUID importId) { return findAllByProperty("import_id", importId); } + public void deleteByImportId(UUID importId) { + deleteByProperty("import_id", importId); + } + } diff --git a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java b/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java index ccb0560eeb..7ac8c49228 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java @@ -55,5 +55,8 @@ public void deleteEnrolleeById(UUID id) { } } + public void deleteByImportId(UUID importId) { + dao.deleteByImportId(importId); + } } diff --git a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java b/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java index a73c28d098..edb8136ae5 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java @@ -62,4 +62,11 @@ public void deleteEnrolleesByImportId(UUID id) { }); } + @Transactional + public void deleteByStudyEnvId(UUID studyEnvironmentId) { + List imports = dao.findAllByStudyEnv(studyEnvironmentId); + imports.forEach(dataImport -> importItemService.deleteByImportId(dataImport.getId())); + dao.deleteByStudyEnvId(studyEnvironmentId); + } + } diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java b/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java index dc479e16f6..1c7f070e94 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java @@ -8,7 +8,8 @@ import bio.terra.pearl.core.model.dataimport.ImportItemStatus; import bio.terra.pearl.core.model.dataimport.ImportStatus; import bio.terra.pearl.core.model.dataimport.ImportType; -import bio.terra.pearl.core.model.audit.ResponsibleEntity; +import bio.terra.pearl.core.model.kit.KitRequest; +import bio.terra.pearl.core.model.kit.KitType; import bio.terra.pearl.core.model.participant.Enrollee; import bio.terra.pearl.core.model.participant.ParticipantUser; import bio.terra.pearl.core.model.participant.PortalParticipantUser; @@ -23,9 +24,12 @@ import bio.terra.pearl.core.service.dataimport.ImportService; import bio.terra.pearl.core.service.exception.internal.InternalServerException; import bio.terra.pearl.core.service.export.formatters.module.EnrolleeFormatter; +import bio.terra.pearl.core.service.export.formatters.module.KitRequestFormatter; import bio.terra.pearl.core.service.export.formatters.module.ParticipantUserFormatter; import bio.terra.pearl.core.service.export.formatters.module.ProfileFormatter; import bio.terra.pearl.core.service.export.formatters.module.SurveyFormatter; +import bio.terra.pearl.core.service.kit.KitRequestDto; +import bio.terra.pearl.core.service.kit.KitRequestService; import bio.terra.pearl.core.service.participant.EnrolleeService; import bio.terra.pearl.core.service.participant.ParticipantUserService; import bio.terra.pearl.core.service.participant.PortalParticipantUserService; @@ -92,6 +96,7 @@ public class EnrolleeImportService { private final TimeShiftPopulateDao timeShiftPopulateDao; private final ImportService importService; private final ImportItemService importItemService; + private final KitRequestService kitRequestService; private final char CSV_DELIMITER = ','; private final char TSV_DELIMITER = '\t'; @@ -100,7 +105,7 @@ public EnrolleeImportService(RegistrationService registrationService, Enrollment SurveyResponseService surveyResponseService, ParticipantTaskService participantTaskService, PortalService portalService, ImportService importService, ImportItemService importItemService, SurveyTaskDispatcher surveyTaskDispatcher, TimeShiftPopulateDao timeShiftPopulateDao, EnrolleeService enrolleeService, ParticipantUserService participantUserService, - PortalParticipantUserService portalParticipantUserService) { + PortalParticipantUserService portalParticipantUserService, KitRequestService kitRequestService) { this.registrationService = registrationService; this.enrollmentService = enrollmentService; this.profileService = profileService; @@ -115,6 +120,7 @@ public EnrolleeImportService(RegistrationService registrationService, Enrollment this.enrolleeService = enrolleeService; this.participantUserService = participantUserService; this.portalParticipantUserService = portalParticipantUserService; + this.kitRequestService = kitRequestService; } @Transactional @@ -144,7 +150,7 @@ public Import importEnrollees(String portalShortcode, String studyShortcode, Stu Enrollee enrollee = null; ImportItem importItem; try { - enrollee = importEnrollee(portalShortcode, studyShortcode, studyEnv, enrolleeMap, exportOptions); + enrollee = importEnrollee(portalShortcode, studyShortcode, studyEnv, enrolleeMap, exportOptions, adminId); importItem = ImportItem.builder() .createdEnrolleeId(enrollee.getId()) .importId(dataImport.getId()) @@ -204,7 +210,7 @@ public List> generateImportMaps(InputStream in, ImportFileFo } } - public Enrollee importEnrollee(String portalShortcode, String studyShortcode, StudyEnvironment studyEnv, Map enrolleeMap, ExportOptions exportOptions) { + public Enrollee importEnrollee(String portalShortcode, String studyShortcode, StudyEnvironment studyEnv, Map enrolleeMap, ExportOptions exportOptions, UUID adminId) { /** while importing handle update for existing import if same enrolle: update enrollee if same participant & same portal.. new enrollee & same profile @@ -215,7 +221,6 @@ public Enrollee importEnrollee(String portalShortcode, String studyShortcode, St DataAuditInfo.systemProcessName(getClass(), "importEnrollee") ).build(); - // ParticipantUserFormatter participantUserFormatter = new ParticipantUserFormatter(exportOptions); final ParticipantUser participantUserInfo = participantUserFormatter.fromStringMap(studyEnv.getId(), enrolleeMap); @@ -226,6 +231,9 @@ public Enrollee importEnrollee(String portalShortcode, String studyShortcode, St /** now update the profile */ Profile profile = importProfile(enrolleeMap, regResult.profile(), exportOptions, studyEnv, auditInfo); + /** populate kit_requests */ + importKitRequests(enrolleeMap, adminId, enrollee); + importSurveyResponses(portalShortcode, enrolleeMap, exportOptions, studyEnv, regResult.portalParticipantUser(), enrollee, auditInfo); /** restore email */ @@ -234,6 +242,33 @@ public Enrollee importEnrollee(String portalShortcode, String studyShortcode, St return enrollee; } + private void importKitRequests(Map enrolleeMap, UUID adminId, Enrollee enrollee) { + new KitRequestFormatter().listFromStringMap(enrolleeMap).stream().map( + kitRequestDto -> kitRequestService.create(convertKitRequestDto(adminId, enrollee, kitRequestDto))).toList(); + } + + private KitRequest convertKitRequestDto( + UUID adminUserId, + Enrollee enrollee, + KitRequestDto kitRequestDto) { + + KitType kitType = kitRequestService.lookupKitTypeByName(kitRequestDto.getKitType().getName()); + return KitRequest.builder() + .creatingAdminUserId(adminUserId) + .enrolleeId(enrollee.getId()) + .status(kitRequestDto.getStatus()) + .sentToAddress(kitRequestDto.getSentToAddress()) + .skipAddressValidation(kitRequestDto.isSkipAddressValidation()) + .kitTypeId(kitType.getId()) + .createdAt(kitRequestDto.getCreatedAt()) + .labeledAt(kitRequestDto.getLabeledAt()) + .sentAt(kitRequestDto.getSentAt()) + .receivedAt(kitRequestDto.getReceivedAt()) + .trackingNumber(kitRequestDto.getTrackingNumber()) + .returnTrackingNumber(kitRequestDto.getReturnTrackingNumber()) + .build(); + } + private RegistrationService.RegistrationResult registerIfNeeded(String portalShortcode, StudyEnvironment studyEnv, ParticipantUser participantUserInfo) { return portalParticipantUserService.findOne(participantUserInfo.getUsername(), portalShortcode, studyEnv.getEnvironmentName()) .map(ppUser -> new RegistrationService.RegistrationResult( @@ -322,6 +357,6 @@ protected SurveyResponse importSurveyResponse(UUID portalId, SurveyFormatter for } // we're not worrying about dating the response yet return surveyResponseService.updateResponse(response, new ResponsibleEntity(DataAuditInfo.systemProcessName(getClass(), "importSurveyResponse")), - ppUser, enrollee, relatedTask.getId(), portalId).getResponse(); + ppUser, enrollee, relatedTask.getId(), portalId).getResponse(); } } diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitRequestTypeFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitRequestTypeFormatter.java deleted file mode 100644 index 0cf72bba30..0000000000 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitRequestTypeFormatter.java +++ /dev/null @@ -1,17 +0,0 @@ -package bio.terra.pearl.core.service.export.formatters.item; - -import bio.terra.pearl.core.service.export.DataValueExportType; -import bio.terra.pearl.core.service.kit.KitRequestDto; - -public class KitRequestTypeFormatter extends PropertyItemFormatter { - public KitRequestTypeFormatter() { - super("kitType", KitRequestDto.class); - this.baseColumnKey = "kitType"; - this.dataType = DataValueExportType.STRING; - } - - @Override - public String getExportString(KitRequestDto bean) { - return bean.getKitType().getName(); - } -} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitTypeFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitTypeFormatter.java new file mode 100644 index 0000000000..cf86c74637 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/KitTypeFormatter.java @@ -0,0 +1,33 @@ +package bio.terra.pearl.core.service.export.formatters.item; + +import bio.terra.pearl.core.model.kit.KitType; +import bio.terra.pearl.core.service.export.DataValueExportType; +import bio.terra.pearl.core.service.kit.KitRequestDto; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.beanutils.PropertyUtils; + +@Slf4j +public class KitTypeFormatter extends PropertyItemFormatter { + public KitTypeFormatter() { + super("kitType", KitRequestDto.class); + this.baseColumnKey = "kitType"; + this.dataType = DataValueExportType.STRING; + } + + @Override + public String getExportString(KitRequestDto bean) { + return bean.getKitType().getName(); + } + + @Override + public void importValueToBean(KitRequestDto bean, String exportString) { + if (exportString != null) { + try { + PropertyUtils.setNestedProperty(bean, getPropertyName(), KitType.builder().name(exportString).build()); + } catch (Exception e) { + log.warn("error setting property " + getPropertyName(), e); + } + } + } + +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/KitRequestFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/KitRequestFormatter.java index 1746458879..0eb9518a9c 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/KitRequestFormatter.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/KitRequestFormatter.java @@ -1,10 +1,13 @@ package bio.terra.pearl.core.service.export.formatters.module; +import bio.terra.pearl.core.model.kit.KitRequestStatus; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.formatters.item.KitRequestTypeFormatter; +import bio.terra.pearl.core.service.export.formatters.item.KitTypeFormatter; import bio.terra.pearl.core.service.export.formatters.item.PropertyItemFormatter; import bio.terra.pearl.core.service.kit.KitRequestDto; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -14,14 +17,15 @@ public class KitRequestFormatter extends ModuleFormatter> { private static final String KIT_REQUEST_MODULE_NAME = "sample_kit"; private static final List KIT_REQUEST_INCLUDED_PROPERTIES = - List.of("status", "sentToAddress", "sentAt", "receivedAt"); + List.of("status", "sentToAddress", "sentAt", "receivedAt", "createdAt", "labeledAt", + "trackingNumber", "returnTrackingNumber", "skipAddressValidation"); public KitRequestFormatter() { itemFormatters = KIT_REQUEST_INCLUDED_PROPERTIES.stream() .map(propName -> new PropertyItemFormatter(propName, KitRequestDto.class)) .collect(Collectors.toList()); // we have to handle kitType separately because we'll need to match it to the kitType name - itemFormatters.add(new KitRequestTypeFormatter()); + itemFormatters.add(new KitTypeFormatter()); moduleName = KIT_REQUEST_MODULE_NAME; displayName = "Sample kit"; } @@ -43,4 +47,38 @@ public Map toStringMap(EnrolleeExportData enrolleeData) { maxNumRepeats = Math.max(maxNumRepeats, sortedKitRequests.size()); return allKitMap; } + + public List listFromStringMap(Map enrolleeMap) { + List kitRequests = new ArrayList<>(); + int requestNum = 1; + KitRequestDto kitRequestDto = getKitRequestDto(enrolleeMap, requestNum); + while (kitRequestDto != null) { + kitRequests.add(kitRequestDto); + requestNum++; + kitRequestDto = getKitRequestDto(enrolleeMap, requestNum); + } + return kitRequests; + } + + private KitRequestDto getKitRequestDto(Map enrolleeMap, int requestNum) { + KitRequestDto kitRequestDto = null; + for (PropertyItemFormatter itemFormatter : itemFormatters) { + String columnName = getColumnKey(itemFormatter, false, null, requestNum); + String stringVal = enrolleeMap.get(columnName); + if (StringUtils.isEmpty(stringVal)) { + continue; + } + if (kitRequestDto == null) { + kitRequestDto = new KitRequestDto(); + } + if (columnName.contains(".status")) { + //enum lookup + kitRequestDto.setStatus(KitRequestStatus.valueOf(stringVal)); + } else { + itemFormatter.importValueToBean(kitRequestDto, stringVal); + } + } + return kitRequestDto; + } + } diff --git a/core/src/main/java/bio/terra/pearl/core/service/kit/KitRequestService.java b/core/src/main/java/bio/terra/pearl/core/service/kit/KitRequestService.java index 03b8fe2ee8..a32f712a0c 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/kit/KitRequestService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/kit/KitRequestService.java @@ -20,7 +20,12 @@ import bio.terra.pearl.core.service.CrudService; import bio.terra.pearl.core.service.exception.NotFoundException; import bio.terra.pearl.core.service.exception.internal.InternalServerException; -import bio.terra.pearl.core.service.kit.pepper.*; +import bio.terra.pearl.core.service.kit.pepper.PepperApiException; +import bio.terra.pearl.core.service.kit.pepper.PepperDSMClientWrapper; +import bio.terra.pearl.core.service.kit.pepper.PepperKit; +import bio.terra.pearl.core.service.kit.pepper.PepperKitAddress; +import bio.terra.pearl.core.service.kit.pepper.PepperKitStatus; +import bio.terra.pearl.core.service.kit.pepper.PepperParseException; import bio.terra.pearl.core.service.participant.EnrolleeService; import bio.terra.pearl.core.service.participant.PortalParticipantUserService; import bio.terra.pearl.core.service.participant.ProfileService; @@ -83,7 +88,7 @@ public KitRequestService(KitRequestDao dao, * Throws PepperApiException if the Pepper API request failed */ public KitRequestDto requestKit(AdminUser adminUser, String studyShortcode, Enrollee enrollee, KitRequestCreationDto kitRequestCreationDto) - throws PepperApiException { + throws PepperApiException { // create and save kit request if (enrollee.getProfileId() == null) { throw new IllegalArgumentException("No profile for enrollee: " + enrollee.getShortcode()); @@ -113,7 +118,8 @@ public KitRequestDto requestKit(AdminUser adminUser, String studyShortcode, Enro } - public record KitRequestCreationDto(String kitType, boolean skipAddressValidation) {} + public record KitRequestCreationDto(String kitType, boolean skipAddressValidation) { + } /** * Collect the address fields sent to Pepper with a kit request. This is not the full DSM request, just the address @@ -147,15 +153,15 @@ public List findByEnrollee(Enrollee enrollee) { */ public Map> findByEnrollees(Collection enrollees) { Map idToEnrollee = - enrollees.stream().collect(Collectors.toMap(Enrollee::getId, Function.identity())); + enrollees.stream().collect(Collectors.toMap(Enrollee::getId, Function.identity())); Map> idToKitRequest = dao.findByEnrolleeIds(idToEnrollee.keySet()); Map kitTypeMap = getKitTypeMap(); return idToKitRequest.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> createKitRequestDto(entry.getValue(), kitTypeMap, idToEnrollee) - )); + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> createKitRequestDto(entry.getValue(), kitTypeMap, idToEnrollee) + )); } /** @@ -167,22 +173,22 @@ public Collection getKitsByStudyEnvironment(StudyEnvironment stud } protected List createKitRequestDto(List kitRequests, - Map kitTypeMap, - Map enrollees + Map kitTypeMap, + Map enrollees ) { List kitRequestDto = new ArrayList<>(); kitRequests.forEach(kit -> { String enrolleeShortcode = enrollees.get(kit.getEnrolleeId()).getShortcode(); KitRequestDto requestDetails = - new KitRequestDto(kit, kitTypeMap.get(kit.getKitTypeId()), enrolleeShortcode, objectMapper); + new KitRequestDto(kit, kitTypeMap.get(kit.getKitTypeId()), enrolleeShortcode, objectMapper); kitRequestDto.add(requestDetails); }); return kitRequestDto; } - protected Map getKitTypeMap() { + protected Map getKitTypeMap() { return kitTypeDao.findAll().stream() - .collect(Collectors.toMap(KitType::getId, Function.identity())); + .collect(Collectors.toMap(KitType::getId, Function.identity())); } /** @@ -197,7 +203,7 @@ public void syncKitStatusFromPepper(UUID kitId) throws PepperParseException, Pep StudyEnvironmentConfig studyEnvironmentConfig = studyEnvironmentConfigService.findByStudyEnvironmentId(enrollee.getStudyEnvironmentId()); PepperKit pepperKitStatus = pepperDSMClientWrapper.fetchKitStatus(studyEnvironmentConfig, kitId); saveKitStatus(kitRequest, pepperKitStatus, Instant.now()); - } + } /** * Query Pepper for all in-progress kits and update the cached status in Juniper. This is intended to be called as a @@ -225,8 +231,8 @@ public void syncAllKitStatusesFromPepper() { StudyEnvironmentConfig studyEnvironmentConfig = studyEnvironmentConfigService.find(studyEnv.getStudyEnvironmentConfigId()).orElseThrow(); Study study = studies.stream().filter(s -> s.getId().equals(studyEnv.getStudyId())).findFirst().orElseThrow(); try { - Collection pepperKits = pepperDSMClientWrapper.fetchKitStatusByStudy(study.getShortcode(), studyEnvironmentConfig); - syncKitStatusesForStudyEnv(study.getShortcode(), studyEnv.getEnvironmentName(), pepperKits); + Collection pepperKits = pepperDSMClientWrapper.fetchKitStatusByStudy(study.getShortcode(), studyEnvironmentConfig); + syncKitStatusesForStudyEnv(study.getShortcode(), studyEnv.getEnvironmentName(), pepperKits); } catch (PepperParseException | PepperApiException e) { // if one sync fails, keep trying others in case the failure is just isolated unexpected data log.error("kit status sync failed for study %s".formatted(study.getShortcode()), e); @@ -286,8 +292,10 @@ public void deleteByEnrolleeId(UUID enrolleeId, Set cascade) { } } - /** Just creates the object -- does not communicate with pepper or save to database. The created - * object will have an id so that external requests will be sent on it. */ + /** + * Just creates the object -- does not communicate with pepper or save to database. The created + * object will have an id so that external requests will be sent on it. + */ public KitRequest assemble( AdminUser adminUser, Enrollee enrollee, @@ -307,6 +315,10 @@ public KitRequest assemble( return kitRequest; } + public KitType lookupKitTypeByName(String kitTypeName) { + return kitTypeDao.findByName(kitTypeName).orElseThrow(() -> new NotFoundException("KitType not found")); + } + protected String stringifyPepperAddress(PepperKitAddress kitAddress) { try { return objectMapper.writeValueAsString(kitAddress); @@ -379,11 +391,12 @@ protected Enrollee getEnrollee(KitRequest kitRequest) { protected Map getEnrollees(List kitRequests) { List enrolleeIds = kitRequests.stream().map(KitRequest::getEnrolleeId).toList(); return enrolleeService.findAll(enrolleeIds).stream() - .collect(Collectors.toMap(Enrollee::getId, Function.identity())); + .collect(Collectors.toMap(Enrollee::getId, Function.identity())); } /** * Parse a Pepper date-time string into an Instant. Returns null if the date string is null or empty. + * * @throws java.time.format.DateTimeParseException if the string is not a valid date-time */ protected Instant parsePepperDateTime(String dateTimeString) { diff --git a/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java b/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java index b4278c9db5..d650706502 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java @@ -11,6 +11,7 @@ import bio.terra.pearl.core.model.survey.StudyEnvironmentSurvey; import bio.terra.pearl.core.service.CascadeProperty; import bio.terra.pearl.core.service.CrudService; +import bio.terra.pearl.core.service.dataimport.ImportService; import bio.terra.pearl.core.service.datarepo.DataRepoJobService; import bio.terra.pearl.core.service.datarepo.DatasetService; import bio.terra.pearl.core.service.exception.NotFoundException; @@ -38,6 +39,7 @@ public class StudyEnvironmentService extends CrudService findByStudy(UUID studyId) { @@ -115,6 +119,7 @@ public void delete(UUID studyEnvironmentId, Set cascade) { withdrawnEnrolleeDao.deleteByStudyEnvironmentId(studyEnvironmentId); adminTaskService.deleteByStudyEnvironmentId(studyEnvironmentId, null); studyEnvironmentKitTypeService.deleteByStudyEnvironmentId(studyEnvironmentId, cascade); + importService.deleteByStudyEnvId(studyEnvironmentId); dao.delete(studyEnvironmentId); if (studyEnv.getStudyEnvironmentConfigId() != null) { studyEnvironmentConfigService.delete(studyEnv.getStudyEnvironmentConfigId()); diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java index 38a926d7b1..658a6d7219 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java @@ -9,6 +9,8 @@ import bio.terra.pearl.core.model.dataimport.Import; import bio.terra.pearl.core.model.dataimport.ImportItem; import bio.terra.pearl.core.model.dataimport.ImportStatus; +import bio.terra.pearl.core.model.kit.KitRequestStatus; +import bio.terra.pearl.core.model.kit.KitType; import bio.terra.pearl.core.model.participant.Enrollee; import bio.terra.pearl.core.model.participant.ParticipantUser; import bio.terra.pearl.core.model.participant.Profile; @@ -20,6 +22,8 @@ import bio.terra.pearl.core.service.dataimport.ImportFileFormat; import bio.terra.pearl.core.service.dataimport.ImportItemService; import bio.terra.pearl.core.service.dataimport.ImportService; +import bio.terra.pearl.core.service.kit.KitRequestDto; +import bio.terra.pearl.core.service.kit.KitRequestService; import bio.terra.pearl.core.service.participant.EnrolleeService; import bio.terra.pearl.core.service.participant.ParticipantUserService; import bio.terra.pearl.core.service.participant.ProfileService; @@ -39,6 +43,7 @@ import java.io.ByteArrayInputStream; import java.time.Instant; import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; @@ -47,6 +52,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; @Slf4j public class EnrolleeImportServiceTests extends BaseSpringBootTest { @@ -74,11 +80,22 @@ public class EnrolleeImportServiceTests extends BaseSpringBootTest { private ImportService importService; @Autowired private ImportItemService importItemService; + @Autowired + KitRequestService kitRequestService; public DataImportSetUp setup(TestInfo info, String csvString) { StudyEnvironmentFactory.StudyEnvironmentBundle bundle = studyEnvironmentFactory.buildBundle(getTestName(info), EnvironmentName.irb); AdminUser adminUser = adminUserFactory.builder(getTestName(info)).build(); AdminUser savedAdmin = adminUserService.create(adminUser); + + //create survey + Survey survey = surveyFactory.buildPersisted(surveyFactory.builderWithDependencies(getTestName(info)) + .content("{\"pages\":[{\"elements\":[{\"type\":\"text\",\"name\":\"diagnosis\",\"title\":\"What is your diagnosis?\"}]}]}") + .stableId("medical_history") + .portalId(bundle.getPortal().getId())); + ; + surveyFactory.attachToEnv(survey, bundle.getStudyEnv().getId(), true); + return new DataImportSetUp().builder() .bundle(bundle) .adminUser(adminUser) @@ -92,16 +109,15 @@ public DataImportSetUp setup(TestInfo info, String csvString) { @Transactional public void testImportEnrolleesCSV(TestInfo info) { String csvString = """ - column1,column2,column3,account.username,account.createdAt,enrollee.createdAt,profile.birthDate - a,b,c,userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1980-10-10" - x,y,z,userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM" + account.username,account.createdAt,enrollee.createdAt,profile.birthDate,medical_history.diagnosis + userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1980-10-10","sick" + userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM" """; - DataImportSetUp setupData = setup(info, csvString); Import dataImport = doImport(setupData.bundle, csvString, setupData.savedAdmin, ImportFileFormat.CSV); UUID studyEnvId = setupData.bundle.getStudyEnv().getId(); List imports = dataImport.getImportItems(); - verifyImport(dataImport); + verifyImport(dataImport, 2); /*create participantUser, enrollee, profile with expected data to assert*/ Enrollee enrolleeExpected = new Enrollee(); @@ -120,15 +136,16 @@ public void testImportEnrolleesCSV(TestInfo info) { userExpected2.setUsername("userName2"); Profile profileExpected2 = new Profile(); verifyParticipant(imports.get(1), studyEnvId, userExpected2, enrolleeExpected2, profileExpected2); + verifySurveyQuestionAnswer(imports.get(0), "medical_history", "diagnosis", "sick"); } @Test @Transactional public void testImportEnrolleeUpdateCSV(TestInfo info) { String csvString = """ - column1,column2,column3,account.username,account.createdAt,enrollee.createdAt,profile.birthDate - a,b,c,userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1980-10-10" - x,y,z,userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM" + account.username,account.createdAt,enrollee.createdAt,profile.birthDate,sample_kit.status,sample_kit.createdAt,sample_kit.sentToAddress,sample_kit.kitType,medical_history.diagnosis + userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1980-10-10","SENT","2024-05-19 01:10PM","{"firstName":"SS","lastName":"LN1","street1":"320 Charles Street","city":"Cambridge","state":"MA","postalCode":"02141","country":"US"}","SALIVA","sick" + userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM" """; DataImportSetUp setupData = setup(info, csvString); Import dataImport = doImport(setupData.bundle, csvString, setupData.savedAdmin, ImportFileFormat.CSV); @@ -137,14 +154,17 @@ public void testImportEnrolleeUpdateCSV(TestInfo info) { ParticipantUser user = participantUserService.find(imports.get(0).getCreatedParticipantUserId()).orElseThrow(); Enrollee enrollee = enrolleeService.findByParticipantUserIdAndStudyEnvId(user.getId(), studyEnvId).orElseThrow(); + //verify survey + verifySurveyQuestionAnswer(imports.get(0), "medical_history", "diagnosis", "sick"); + /*now try update*/ String csvStringUpdate = """ - account.username,account.createdAt,enrollee.createdAt,profile.birthDate - userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1982-10-10" - userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM","1990-10-10" + account.username,account.createdAt,enrollee.createdAt,profile.birthDate,medical_history.diagnosis + userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1982-10-10","healthy" + userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM","1990-10-10","not healthy" """; Import dataImportUpdate = doImport(setupData.bundle, csvStringUpdate, setupData.savedAdmin, ImportFileFormat.CSV); - verifyImport(dataImportUpdate); + verifyImport(dataImportUpdate, 2); /*create participantUser, enrollee, profile with expected data to assert*/ ParticipantUser userExpected = new ParticipantUser(); userExpected.setCreatedAt(Instant.parse("2024-05-09T13:37:00Z")); @@ -155,6 +175,7 @@ public void testImportEnrolleeUpdateCSV(TestInfo info) { profileExpected.setId(enrollee.getProfileId()); //should be same profile profileExpected.setBirthDate(LocalDate.parse("1982-10-10")); verifyParticipant(imports.get(0), studyEnvId, userExpected, enrolleeExpected, profileExpected); + verifySurveyQuestionAnswer(dataImportUpdate.getImportItems().get(0), "medical_history", "diagnosis", "healthy"); //enrollee2 ParticipantUser user2 = participantUserService.find(imports.get(1).getCreatedParticipantUserId()).orElseThrow(); @@ -168,6 +189,64 @@ public void testImportEnrolleeUpdateCSV(TestInfo info) { profileExpected2.setId(enrollee2.getProfileId()); //should be same profile profileExpected2.setBirthDate(LocalDate.parse("1990-10-10")); verifyParticipant(imports.get(1), studyEnvId, userExpected2, enrolleeExpected2, profileExpected2); + verifySurveyQuestionAnswer(dataImportUpdate.getImportItems().get(1), "medical_history", "diagnosis", "not healthy"); + } + + @Test + @Transactional + public void testImportEnrolleeSingleKitRequest(TestInfo info) { + String csvStringSingleKit = """ + account.username,account.createdAt,enrollee.createdAt,profile.birthDate,sample_kit.status,sample_kit.createdAt,sample_kit.sentAt,sample_kit.trackingNumber,sample_kit.sentToAddress,sample_kit.kitType + userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1980-10-10","SENT","2024-05-09 10:10AM","2024-05-19 01:38PM","KITTRACKNUMBER12345","{\"firstName\":\"SS\",\"lastName\":\"LN1\",\"street1\":\"320 Charles Street\",\"city\":\"Cambridge\",\"state\":\"MA\",\"postalCode\":\"02141\",\"country\":\"US\"}","SALIVA" + userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM" + """; + + DataImportSetUp setupData = setup(info, csvStringSingleKit); + Import dataImport = doImport(setupData.bundle, csvStringSingleKit, setupData.savedAdmin, ImportFileFormat.CSV); + verifyImport(dataImport, 2); + + List kitRequestDtoList = new ArrayList<>(); + KitType salivaKit = KitType.builder().name("SALIVA").build(); + KitRequestDto kitRequestDto = KitRequestDto.builder() + .kitType(salivaKit) + .status(KitRequestStatus.SENT) + .trackingNumber("KITTRACKNUMBER12345") + .createdAt(Instant.parse("2024-05-09T10:10:00Z")) + .build(); + kitRequestDtoList.add(kitRequestDto); + verifyKitRequests(dataImport.getImportItems().get(0), kitRequestDtoList); + } + + @Test + @Transactional + public void testImportEnrolleeMultipleKitRequests(TestInfo info) { + String csvStringMultipleKits = """ + account.username,account.createdAt,enrollee.createdAt,profile.birthDate,sample_kit.status,sample_kit.createdAt,sample_kit.sentAt,sample_kit.trackingNumber,sample_kit.sentToAddress,sample_kit.kitType,medical_history.diagnosis,sample_kit.2.status,sample_kit.2.createdAt,sample_kit.2.sentAt,sample_kit.2.receivedAt,sample_kit.2.trackingNumber,sample_kit.2.sentToAddress,sample_kit.2.kitType + userName1,"2024-05-09 01:37PM","2024-05-09 01:38PM","1980-10-10","SENT","2024-05-09 10:10AM","2024-05-19 01:38PM","KITTRACKNUMBER_1","{\"firstName\":\"SS\",\"lastName\":\"LN1\",\"street1\":\"320 Charles Street\",\"city\":\"Cambridge\",\"state\":\"MA\",\"postalCode\":\"02141\",\"country\":\"US\"}","SALIVA", "sick","RECEIVED","2024-05-21 11:10AM","2024-05-22 01:38PM","2024-05-25 01:10AM","KITTRACKNUMBER_2","{\"firstName\":\"SS2\",\"street1\":\"320 Charles Street\",\"city\":\"Cambridge\"}","SALIVA" + userName2,"2024-05-11 10:00AM","2024-05-11 10:00AM" + """; + + DataImportSetUp setupData = setup(info, csvStringMultipleKits); + List kitRequestDtoList = new ArrayList<>(); + KitType salivaKit = KitType.builder().name("SALIVA").build(); + KitRequestDto kitRequestDto = KitRequestDto.builder() + .kitType(salivaKit) + .status(KitRequestStatus.SENT) + .trackingNumber("KITTRACKNUMBER_1") + .createdAt(Instant.parse("2024-05-09T10:10:00Z")) + .build(); + KitRequestDto kitRequestDto2 = KitRequestDto.builder() + .kitType(salivaKit) + .status(KitRequestStatus.RECEIVED) + .trackingNumber("KITTRACKNUMBER_2") + .createdAt(Instant.parse("2024-05-21T11:10:00Z")) + .build(); + kitRequestDtoList.add(kitRequestDto); + kitRequestDtoList.add(kitRequestDto2); + + Import dataImport = doImport(setupData.bundle, csvStringMultipleKits, setupData.savedAdmin, ImportFileFormat.CSV); + verifyImport(dataImport, 2); + verifyKitRequests(dataImport.getImportItems().get(0), kitRequestDtoList); } @Test @@ -193,7 +272,7 @@ public void testImportEnrolleeUpdateDiffPortalCSV(TestInfo info) { """; DataImportSetUp setupData2 = setup(info, csvStringPortal2); Import dataImportUpdate = doImport(setupData2.bundle, csvStringPortal2, setupData2.savedAdmin, ImportFileFormat.CSV); - verifyImport(dataImportUpdate); + verifyImport(dataImportUpdate, 2); ImportItem importItem = dataImportUpdate.getImportItems().get(0); ParticipantUser userUpd = participantUserService.find(importItem.getCreatedParticipantUserId()).orElseThrow(); Enrollee enrolleeUpd = enrolleeService.findByParticipantUserIdAndStudyEnvId(userUpd.getId(), studyEnvId).orElseThrow(); @@ -270,7 +349,7 @@ public void testBaseEnrolleeImport(TestInfo info) { bundle.getStudy().getShortcode(), bundle.getStudyEnv(), enrolleeMap, - new ExportOptions()); + new ExportOptions(), null); ParticipantUser user = participantUserService.findOne(username, bundle.getStudyEnv().getEnvironmentName()).orElseThrow(); Enrollee enrollee = enrolleeService.findByParticipantUserIdAndStudyEnvId(user.getId(), bundle.getStudyEnv().getId()).orElseThrow(); assertThat(enrollee.isSubject(), equalTo(true)); @@ -294,7 +373,7 @@ public void testEnrolleeProfileImport(TestInfo info) { bundle.getStudy().getShortcode(), bundle.getStudyEnv(), enrolleeMap, - new ExportOptions()); + new ExportOptions(), null); Profile profile = profileService.loadWithMailingAddress(enrolle.getProfileId()).orElseThrow(); assertThat(profile.getGivenName(), equalTo("Alex")); assertThat(profile.getBirthDate(), equalTo(LocalDate.of(1998, 5, 14))); @@ -354,7 +433,7 @@ public void testSurveyResponseImport(TestInfo info) { bundle.getStudy().getShortcode(), bundle.getStudyEnv(), enrolleeMap, - new ExportOptions()); + new ExportOptions(), null); // confirm a task got created for the enrollee, and the task is complete List tasks = participantTaskService.findByEnrolleeId(enrollee.getId()); assertThat(tasks, hasSize(1)); @@ -388,13 +467,15 @@ private void verifyParticipant(ImportItem importItem, UUID studyEnvId, } } - private void verifyImport(Import dataImport) { + private void verifyImport(Import dataImport, int importItemCount) { Import dataImportQueried = importService.find(dataImport.getId()).get(); assertThat(dataImport, is(dataImportQueried)); assertThat(dataImport.getStatus(), is(ImportStatus.DONE)); importItemService.attachImportItems(dataImport); List imports = dataImport.getImportItems(); - assertThat(imports, hasSize(2)); + assertThat(imports, hasSize(importItemCount)); + long enrolleeCount = imports.stream().filter(importItem -> importItem.getCreatedEnrolleeId() != null).count(); + assertThat(enrolleeCount, equalTo(Long.valueOf(importItemCount))); } private Import doImport(StudyEnvironmentFactory.StudyEnvironmentBundle bundle, String csvString, AdminUser admin, ImportFileFormat fileType) { @@ -406,6 +487,32 @@ private Import doImport(StudyEnvironmentFactory.StudyEnvironmentBundle bundle, S admin.getId(), fileType); } + private void verifySurveyQuestionAnswer(ImportItem importItem, String surveyStableId, String questionStableId, String questionAnswer) { + List tasks = participantTaskService.findByEnrolleeId(importItem.getCreatedEnrolleeId()); + assertThat(tasks, hasSize(1)); + List answers = answerService.findByEnrolleeAndSurvey(importItem.getCreatedEnrolleeId(), surveyStableId); + assertThat(answers, hasSize(1)); + Answer thisAnswer = answers.stream().filter(answer -> answer.getQuestionStableId().equals(questionStableId)) + .findFirst().get(); + assertThat(thisAnswer.getStringValue(), equalTo(questionAnswer)); + } + + private void verifyKitRequests(ImportItem importItem, List expectedKitRequests) { + + List kitRequestDtos = kitRequestService.findByEnrollee(enrolleeService.find(importItem.getCreatedEnrolleeId()).get()); + assertThat(kitRequestDtos.size(), equalTo(expectedKitRequests.size())); + for (int i = 0; i < expectedKitRequests.size(); i++) { + KitRequestDto kitRequestDto = kitRequestDtos.get(i); + KitRequestDto expectedKit = expectedKitRequests.get(i); + + assertThat(kitRequestDto.getKitType().getName(), equalTo(expectedKit.getKitType().getName())); + assertThat(kitRequestDto.getTrackingNumber(), equalTo(expectedKit.getTrackingNumber())); + assertThat(kitRequestDto.getStatus(), equalTo(expectedKit.getStatus())); + assertThat(kitRequestDto.getSentToAddress(), notNullValue()); + assertThat(kitRequestDto.getCreatedAt(), equalTo(expectedKit.getCreatedAt())); + } + } + @AllArgsConstructor @NoArgsConstructor @Builder