From 173f3411c8ea4c8f2eb27bc732ccf8bfe5de8972 Mon Sep 17 00:00:00 2001 From: Manda Wilson <1458628+mandawilson@users.noreply.github.com> Date: Thu, 21 Dec 2023 17:52:56 -0500 Subject: [PATCH] parent 556e9b9032e9cb9fbe7754b05be8be84b12757ed author Manda Wilson <1458628+mandawilson@users.noreply.github.com> 1703199176 -0500 committer Robert Sheridan 1711560265 -0400 upgrade to java 21 switch to genome-nexus-annotation-pipeline that uses new maf repo updated to spring 6, spring batch 5, spring boot 3 to match cbioportal fix typos Updates to AZ-MSKIMPACT to integrate with CDM (#1098) Fix bug in checking for duplicate Mutation Records (#1099) * Check if mutationRecord is duplicated before annotating * Populate mutationMap in loadMutationRecordsFromJson * add addRecordToMap * Remove comments, add local vars for debugging * Remove duplicate MAF variants for AZ * Fix remove-duplicate-maf-variants call * revert whitespace change updates for migrating darwin and crdb to java11 (#1080) pom changes for pulling moved dependencies changes to java args to silence warnings Co-authored-by: cbioportal import user Remove Annotated MAF before Import (#958) * remove annotated MAF to prevent duplicate * Update subset_and_merge_crdb_pdx_studies.py --------- Co-authored-by: Avery Wang Script to combine arbitrary files (#1104) * Script to combine arbitrary files * Modify unit tests to work with script changes * Remove unnecessary column specifier * Fix syntax bug Add sophia script (#1105) * Add sophia script * rename transpose_cna file * Add filter-clinical-arg-functions script * Add az var to correct automation environment * Add correct path to transpose_cna script * Call seq_date function * Add seq_date before filtering columns * syntax fix * Fix call to filter out clinical attribute columns * Fix nonsigned out file path * Automate folder name * directory fixes * remove quotes? * change date formatting * output filepath for duplicate variants script * use az_msk_impact_data_home var * move sophia_data_home to automation environment * Add comments * Change dir structures in sophia script to match new repo structure * Add git operations * Remove test file * Fix dirs for sophia zip command * remove quotes * Zip files before cleanup * move zip step before git push Add script for merging Dremio/SMILE into cmo-access (#1102) - adds cfdna clinical and timeline data from dremio/SMILE - converts patient identifiers using "dmp over cmo" identifier logic from dremio - dremio patient id mapping table export code called to produce mapping table - main script then calls update_cfdna_clinical_sample_patient_ids_via_dremio.sh - merge.py used to combine clinical data from dremio with clinical data from cmo-access - metadata headers added using new script : merge_clinical_metadata_headers_py3.py - other import process flow (similar to other import scripts) followed - error detection step added after debugging for sporadic data loss in results Co-authored-by: Manda Wilson <1458628+mandawilson@users.noreply.github.com> Modify preconsume script to work on one cohort at a time (#1107) Call correct function name add options for logging in for different accounts Preconsume archer-solid-cv4 and add fetch loop (#1129) * Handle archer-solid-cv4 samples * Add loop * move each cohort to its own dir and fix filename switch to genome-nexus-annotation-pipeline that uses new maf repo use updated genome-nexus-annotation-pipeline update version of cmo-pipelines to 1.0.0 Convert BatchConfiguration to new Spring Batch format drop unneeded dependency from redcap removed gdd, updated crdb and ddp batch configs to spring batch 5 removed commons-lang start of converting cvr to spring batch 5 fix cvr fetcher BatchConfiguration fixed redcap pipeline spring batch 5 configuration make spring-batch-integration match batch version Co-authored-by: Manda Wilson <1458628+mandawilson@users.noreply.github.com> drop darwin fetcher (and docs/scripts) --- .circleci/config.yml | 7 +- README.md | 14 - common/pom.xml | 10 +- .../cmo/pipelines/common/util/EmailUtil.java | 8 +- crdb/pom.xml | 12 +- .../ks/crdb/pipeline/BatchConfiguration.java | 97 +- .../ks/crdb/pipeline/CRDBDatasetWriter.java | 8 +- ...RDBPDXClinicalAnnotationMappingWriter.java | 2 +- .../CRDBPDXClinicalPatientWriter.java | 2 +- .../pipeline/CRDBPDXClinicalSampleWriter.java | 2 +- ...DBPDXSourceToDestinationMappingWriter.java | 2 +- .../crdb/pipeline/CRDBPDXTimelineWriter.java | 2 +- .../ks/crdb/pipeline/CRDBSurveyWriter.java | 8 +- cvr/pom.xml | 20 +- .../cmo/pipelines/cvr/BatchConfiguration.java | 382 +-- .../cmo/pipelines/cvr/CVRUtilities.java | 6 +- .../cvr/clinical/CVRClinicalDataWriter.java | 4 +- .../CVRSeqDateClinicalDataWriter.java | 4 +- .../pipelines/cvr/cna/CVRCnaDataWriter.java | 7 +- .../cvr/consume/ConsumeSampleWriter.java | 4 +- .../cvr/genepanel/CVRGenePanelWriter.java | 2 +- .../LinkedMskimpactCaseWriter.java | 5 +- .../cvr/model/CVRCnvIntragenicVariant.java | 8 +- .../pipelines/cvr/model/CVRCnvVariant.java | 2 +- .../cmo/pipelines/cvr/model/CVRData.java | 2 +- .../pipelines/cvr/model/CVRMasterList.java | 10 +- .../pipelines/cvr/model/CVRMergedResult.java | 10 +- .../cmo/pipelines/cvr/model/CVRMetaData.java | 2 +- .../pipelines/cvr/model/CVRRequeueRecord.java | 10 +- .../cmo/pipelines/cvr/model/CVRResult.java | 10 +- .../cmo/pipelines/cvr/model/CVRSegData.java | 2 +- .../cmo/pipelines/cvr/model/CVRSession.java | 10 +- .../cmo/pipelines/cvr/model/CVRSnp.java | 2 +- .../cmo/pipelines/cvr/model/CVRSvVariant.java | 2 +- .../cmo/pipelines/cvr/model/CvrResponse.java | 10 +- .../cvr/model/GMLCnvIntragenicVariant.java | 5 +- .../cmo/pipelines/cvr/model/GMLData.java | 2 +- .../cmo/pipelines/cvr/model/GMLMetaData.java | 2 +- .../cmo/pipelines/cvr/model/GMLResult.java | 10 +- .../cmo/pipelines/cvr/model/GMLSnp.java | 2 +- .../cmo/pipelines/cvr/model/GMLVariant.java | 10 +- .../cvr/mutation/CVRMutationDataReader.java | 20 +- .../cvr/mutation/CVRMutationDataWriter.java | 2 +- .../CVRNonSignedoutMutationDataReader.java | 21 +- .../cvr/mutation/GMLMutationDataReader.java | 21 +- .../cvr/samplelist/CvrSampleListsTasklet.java | 2 +- .../pipelines/cvr/seg/CVRSegDataWriter.java | 4 +- .../cmo/pipelines/cvr/sv/CVRSvDataWriter.java | 5 +- .../cvr/variants/CVRVariantsWriter.java | 2 +- .../cvr/variants/CvrResponseTasklet.java | 2 +- .../cvr/variants/GMLVariantsWriter.java | 3 +- darwin/pom.xml | 109 - .../mskcc/cmo/ks/darwin/DarwinPipeline.java | 134 - .../darwin/pipeline/BatchConfiguration.java | 565 ----- .../darwin/pipeline/DarwinConfiguration.java | 103 - .../darwin/pipeline/DarwinEmailTasklet.java | 101 - .../model/MskimpactBrainSpineClinical.java | 212 -- .../MskimpactBrainSpineCompositeTimeline.java | 110 - .../model/MskimpactBrainSpineTimeline.java | 482 ---- .../model/MskimpactCompositeDemographics.java | 87 - .../model/MskimpactGenieClinical.java | 93 - .../model/MskimpactLatestActivity.java | 257 -- .../model/MskimpactMedicalTherapy.java | 165 -- .../model/MskimpactNAACCRClinical.java | 145 -- .../pipeline/model/MskimpactPathology.java | 113 - .../pipeline/model/MskimpactPathologyDmp.java | 77 - .../model/MskimpactPatientDemographics.java | 510 ---- .../model/MskimpactPatientIcdoRecord.java | 2213 ----------------- .../model/NAACCREthnicityMapping.java | 76 - .../pipeline/model/NAACCRRaceMapping.java | 76 - .../pipeline/model/NAACCRSexMapping.java | 77 - ...kcc_2015_chantClinicalCompositeRecord.java | 46 - .../Skcm_mskcc_2015_chantClinicalRecord.java | 974 -------- ...cc_2015_chantNormalizedClinicalRecord.java | 859 ------- ...cm_mskcc_2015_chantTimelineAdjuvantTx.java | 171 -- ...cm_mskcc_2015_chantTimelineRadTherapy.java | 162 -- .../Skcm_mskcc_2015_chantTimelineRecord.java | 50 - ...cm_mskcc_2015_chantTimelineSystemicTx.java | 162 -- ...MskimpactMedicalTherapyClinicalWriter.java | 99 - .../MskimpactMedicalTherapyProcessor.java | 85 - .../MskimpactMedicalTherapyReader.java | 207 -- ...MskimpactMedicalTherapyTimelineWriter.java | 92 - .../MskimpactBrainSpineClinicalProcessor.java | 59 - .../MskimpactBrainSpineClinicalReader.java | 126 - .../MskimpactBrainSpineClinicalWriter.java | 98 - .../BrainSpineTimelineType.java | 52 - ...ainSpineCompositeToCompositeProcessor.java | 68 - ...neBrainSpineModelToCompositeProcessor.java | 71 - .../MskimpactTimelineBrainSpineReader.java | 118 - .../MskimpactTimelineBrainSpineWriter.java | 114 - .../MskimpactAgeWriter.java | 98 - ...MskimpactPatientDemographicsProcessor.java | 74 - .../MskimpactPatientDemographicsReader.java | 215 -- .../MskimpactPatientDemographicsWriter.java | 106 - .../MskimpactVitalStatusWriter.java | 102 - .../MskimpactGeniePatientProcessor.java | 59 - .../MskimpactGeniePatientReader.java | 98 - .../MskimpactGeniePatientWriter.java | 98 - ...cc_2015_chantClinicalPatientProcessor.java | 83 - ...mskcc_2015_chantClinicalPatientWriter.java | 111 - .../Skcm_mskcc_2015_chantClinicalReader.java | 323 --- ...kcc_2015_chantClinicalSampleProcessor.java | 82 - ..._mskcc_2015_chantClinicalSampleWriter.java | 115 - ...kcm_mskcc_2015_chantTimelineProcessor.java | 64 - .../Skcm_mskcc_2015_chantTimelineReader.java | 134 - .../Skcm_mskcc_2015_chantTimelineWriter.java | 99 - .../pipeline/util/DarwinSampleListUtil.java | 75 - .../ks/darwin/pipeline/util/DarwinUtils.java | 43 - .../pipeline/util/VitalStatusUtils.java | 94 - .../resources/application.properties.EXAMPLE | 64 - .../main/resources/log4j.properties.EXAMPLE | 32 - .../ks/darwin/DarwinTestConfiguration.java | 77 - ...MskimpactTimelineBrainSpineReaderTest.java | 140 -- ddp/ddp_pipeline/pom.xml | 9 +- .../ks/ddp/pipeline/AgeAtSeqDateWriter.java | 4 +- .../ks/ddp/pipeline/BatchConfiguration.java | 94 +- .../cmo/ks/ddp/pipeline/ClinicalWriter.java | 4 +- .../mskcc/cmo/ks/ddp/pipeline/DDPReader.java | 2 +- .../pipeline/SuppNaaccrMappingsWriter.java | 4 +- .../ddp/pipeline/SuppVitalStatusWriter.java | 4 +- .../ks/ddp/pipeline/TimelineChemoWriter.java | 4 +- .../ddp/pipeline/TimelineRadiationWriter.java | 4 +- .../ddp/pipeline/TimelineSurgeryWriter.java | 4 +- .../pipeline/model/AgeAtSeqDateRecord.java | 8 +- .../ks/ddp/pipeline/model/ClinicalRecord.java | 8 +- .../model/SuppNaaccrMappingsRecord.java | 8 +- .../pipeline/model/SuppVitalStatusRecord.java | 8 +- .../pipeline/model/TimelineChemoRecord.java | 8 +- .../model/TimelineRadiationRecord.java | 8 +- .../pipeline/model/TimelineSurgeryRecord.java | 8 +- .../ks/ddp/pipeline/util/DDPUtilsTest.java | 2 +- .../source/internal/DDPRepositoryTest.java | 2 +- ddp/ddp_source/pom.xml | 2 +- ddp/pom.xml | 2 +- docs/DMP-Updates-Workflow.md | 73 - gdd/pom.xml | 51 - .../org/mskcc/cmo/ks/gdd/GDDPipeline.java | 122 - .../ks/gdd/pipeline/BatchConfiguration.java | 93 - .../gdd/pipeline/GDDClassifierProcessor.java | 52 - .../ks/gdd/pipeline/GDDClassifierReader.java | 108 - .../cmo/ks/gdd/pipeline/GDDResultWriter.java | 89 - .../gdd/pipeline/model/GDDClassification.java | 70 - .../ks/gdd/pipeline/model/GDDClassifier.java | 175 -- .../ks/gdd/pipeline/model/GDDEvidence.java | 70 - .../cmo/ks/gdd/pipeline/model/GDDResult.java | 209 -- .../resources/application.properties.EXAMPLE | 2 - gdd/src/main/resources/gdd-sample.json | 76 - gdd/src/main/resources/gdd-schema.json | 67 - gene/pom.xml | 82 - .../org/mskcc/cmo/ks/GeneDataPipeline.java | 97 - .../mskcc/cmo/ks/gene/GeneDataListener.java | 122 - .../mskcc/cmo/ks/gene/GeneDataProcessor.java | 107 - .../org/mskcc/cmo/ks/gene/GeneDataReader.java | 258 -- .../org/mskcc/cmo/ks/gene/GeneDataWriter.java | 93 - .../ks/gene/config/BatchConfiguration.java | 186 -- .../cmo/ks/gene/config/JdbcConfiguration.java | 92 - .../mskcc/cmo/ks/gene/jdbc/DaoGeneJdbc.java | 58 - .../gene/jdbc/internal/DaoGeneJdbcImpl.java | 280 --- .../org/mskcc/cmo/ks/gene/model/Gene.java | 160 -- .../mskcc/cmo/ks/gene/model/GeneAlias.java | 106 - .../mskcc/cmo/ks/gene/util/GeneDataUtil.java | 71 - .../resources/application.properties.EXAMPLE | 8 - .../main/resources/log4j.properties.EXAMPLE | 32 - import-scripts/backup-eks-dbs.sh | 2 +- .../clear_cbioportal_persistence_cache.sh | 3 +- import-scripts/combine_files_py3.py | 127 + .../delete_slow_to_terminate_pods.sh | 2 +- ...etect_samples_with_problematic_metadata.py | 2 +- import-scripts/dmp-import-vars-functions.sh | 9 - .../fetch-ddp-and-import-cmo-access-data.sh | 2 +- import-scripts/fetch-dmp-data-for-import.sh | 11 +- .../filter-clinical-arg-functions.sh | 107 + import-scripts/import-cmo-data-triage.sh | 2 +- import-scripts/import-devdb-data.sh | 2 +- import-scripts/import-hgnc-data.sh | 2 +- .../merge_clinical_metadata_headers_py3.py | 232 ++ import-scripts/merge_ddp_files.py | 119 - ...ge_dremio_clinical_data_into_cmo_access.sh | 401 +++ .../pipelines_eks/automation-environment.sh | 3 + import-scripts/pipelines_eks/mycrontab | 1 + import-scripts/pom.xml | 5 +- import-scripts/precache-portals.py | 2 +- .../preconsume_problematic_samples.sh | 137 +- .../remove-duplicate-maf-variants.py | 89 + .../diff_headers/data_clinical_patient.txt | 0 .../diff_headers/ddp_naaccr.txt | 0 .../diff_headers/ddp_vital_status.txt | 3 +- .../combine_files/diff_headers/expected.txt | 8 + .../duplicates/data_clinical_patient.txt | 0 .../combine_files}/duplicates/ddp_naaccr1.txt | 0 .../combine_files}/duplicates/ddp_naaccr2.txt | 0 .../combine_files}/duplicates/ddp_naaccr3.txt | 0 .../combine_files}/duplicates/expected.txt | 0 .../data_clinical_patient.txt | 0 .../ddp_vital_status1.txt | 0 .../ddp_vital_status2.txt | 0 .../has_metadata_headers/expected.txt | 0 .../data_clinical_patient.txt | 0 .../not_on_masterlist/ddp_naaccr1.txt | 0 .../not_on_masterlist/ddp_naaccr2.txt | 0 .../not_on_masterlist/expected.txt | 0 import-scripts/test-py3/test_combine_files.py | 81 + .../no_patientid/data_clinical_patient.txt | 14 - .../no_patientid/ddp_naaccr.txt | 6 - import-scripts/test/test_merge_ddp_files.py | 84 - import-scripts/transpose_cna_py3.py | 44 + import-scripts/update-az-mskimpact.sh | 188 +- import-scripts/update-sophia-mskimpact.sh | 286 +++ ..._clinical_sample_patient_ids_via_dremio.sh | 169 ++ .../fetch_and_compare_to_redcap_schema.py | 14 +- pom.xml | 22 +- redcap/pom.xml | 6 +- redcap/redcap_pipeline/pom.xml | 11 +- .../redcap/pipeline/BatchConfiguration.java | 202 +- .../pipeline/ClinicalPatientDataWriter.java | 5 +- .../pipeline/ClinicalSampleDataWriter.java | 5 +- .../pipeline/RawClinicalDataWriter.java | 3 +- .../ks/redcap/pipeline/TimelineWriter.java | 3 +- redcap/redcap_source/pom.xml | 4 +- .../redcap/models/OverriddenCancerStudy.java | 2 +- .../models/RedcapAttributeMetadata.java | 2 +- .../redcap/models/RedcapProjectAttribute.java | 2 +- .../cmo/ks/redcap/models/RedcapToken.java | 2 +- .../source/internal/RedcapSessionManager.java | 4 +- 224 files changed, 2335 insertions(+), 15806 deletions(-) delete mode 100644 darwin/pom.xml delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/DarwinPipeline.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/BatchConfiguration.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinConfiguration.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinEmailTasklet.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineClinical.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineCompositeTimeline.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineTimeline.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactCompositeDemographics.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactGenieClinical.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactLatestActivity.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactMedicalTherapy.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactNAACCRClinical.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathology.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathologyDmp.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientDemographics.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientIcdoRecord.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCREthnicityMapping.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRRaceMapping.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRSexMapping.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalCompositeRecord.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalRecord.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantNormalizedClinicalRecord.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineAdjuvantTx.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRadTherapy.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRecord.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineSystemicTx.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyClinicalWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyTimelineWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/BrainSpineTimelineType.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineCompositeToCompositeProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineModelToCompositeProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactAgeWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactVitalStatusWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineProcessor.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineReader.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineWriter.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinSampleListUtil.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinUtils.java delete mode 100644 darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/VitalStatusUtils.java delete mode 100644 darwin/src/main/resources/application.properties.EXAMPLE delete mode 100644 darwin/src/main/resources/log4j.properties.EXAMPLE delete mode 100644 darwin/src/test/java/org/mskcc/cmo/ks/darwin/DarwinTestConfiguration.java delete mode 100644 darwin/src/test/java/org/mskcc/cmo/ks/darwin/MskimpactTimelineBrainSpineReaderTest.java delete mode 100644 gdd/pom.xml delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/GDDPipeline.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/BatchConfiguration.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierProcessor.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierReader.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDResultWriter.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassification.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassifier.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDEvidence.java delete mode 100644 gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDResult.java delete mode 100644 gdd/src/main/resources/application.properties.EXAMPLE delete mode 100644 gdd/src/main/resources/gdd-sample.json delete mode 100644 gdd/src/main/resources/gdd-schema.json delete mode 100644 gene/pom.xml delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/GeneDataPipeline.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataListener.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataProcessor.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataReader.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataWriter.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/config/BatchConfiguration.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/config/JdbcConfiguration.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/DaoGeneJdbc.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/internal/DaoGeneJdbcImpl.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/model/Gene.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/model/GeneAlias.java delete mode 100644 gene/src/main/java/org/mskcc/cmo/ks/gene/util/GeneDataUtil.java delete mode 100644 gene/src/main/resources/application.properties.EXAMPLE delete mode 100644 gene/src/main/resources/log4j.properties.EXAMPLE create mode 100644 import-scripts/combine_files_py3.py create mode 100644 import-scripts/filter-clinical-arg-functions.sh create mode 100755 import-scripts/merge_clinical_metadata_headers_py3.py delete mode 100644 import-scripts/merge_ddp_files.py create mode 100755 import-scripts/merge_dremio_clinical_data_into_cmo_access.sh create mode 100644 import-scripts/remove-duplicate-maf-variants.py rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/diff_headers/data_clinical_patient.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/diff_headers/ddp_naaccr.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/diff_headers/ddp_vital_status.txt (67%) create mode 100644 import-scripts/test-py3/resources/combine_files/diff_headers/expected.txt rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/duplicates/data_clinical_patient.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/duplicates/ddp_naaccr1.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/duplicates/ddp_naaccr2.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/duplicates/ddp_naaccr3.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/duplicates/expected.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/has_metadata_headers/data_clinical_patient.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/has_metadata_headers/ddp_vital_status1.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/has_metadata_headers/ddp_vital_status2.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/has_metadata_headers/expected.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/not_on_masterlist/data_clinical_patient.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/not_on_masterlist/ddp_naaccr1.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/not_on_masterlist/ddp_naaccr2.txt (100%) rename import-scripts/{test/resources/merge_ddp_files => test-py3/resources/combine_files}/not_on_masterlist/expected.txt (100%) create mode 100644 import-scripts/test-py3/test_combine_files.py delete mode 100644 import-scripts/test/resources/merge_ddp_files/no_patientid/data_clinical_patient.txt delete mode 100644 import-scripts/test/resources/merge_ddp_files/no_patientid/ddp_naaccr.txt delete mode 100644 import-scripts/test/test_merge_ddp_files.py create mode 100644 import-scripts/transpose_cna_py3.py create mode 100755 import-scripts/update-sophia-mskimpact.sh create mode 100755 import-scripts/update_cfdna_clinical_sample_patient_ids_via_dremio.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 75f599df3..30a565ac8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,6 +22,9 @@ jobs: - run: name: update apt-get command: sudo apt-get update --fix-missing + - run: + name: update java + command: wget https://download.java.net/java/GA/jdk21.0.2/f2283984656d49d69e91c558476027ac/13/GPL/openjdk-21.0.2_linux-x64_bin.tar.gz; sudo tar -xvf openjdk-21.0.2_linux-x64_bin.tar.gz -C /usr/lib/jvm - run: name: Install setuptools in python 2 command: sudo apt-get -y install python-pip @@ -37,11 +40,11 @@ jobs: # Use mvn clean and package as the standard maven build phase - run: name: Build - command: mvn -B -DskipTests clean package + command: export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2; export PATH=/usr/lib/jvm/jdk-21.0.2/bin:$PATH; mvn -B -DskipTests clean package # Then run your tests! - run: name: Test - command: mvn test + command: export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2; export PATH=/usr/lib/jvm/jdk-21.0.2/bin:$PATH; mvn test # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows diff --git a/README.md b/README.md index 96a2600cc..f6b225488 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ There are these Java components and appliations: - common : a java library of helpful utilities used (as a dependency) in other components - gdd : the "genome directed diagnosis pipeline", which is not currently being maintained (delete?) - crdb : "crdb_fetcher", a pipeline which fetches data from the clinical research database -- darwin : "darwin_fetcher", a pipeline which fetches data from the Darwin DB2 database. In particular, Caisis timeline data. - redcap : "redcap_pipeline", a pipeline which uploads data to or downloads data from the redcap clinical database server - cvr : "cvr_fetcher", a pipeline which downloads samples with identified genomic variants and clinical data from the CVR servers (tumor and germline) - gene : "gene_data_updater", a pipeline which processes a downloaded NCBI human gene info file and encorporates info into the cBioPortal gene table. No longer maintained. (delete?) @@ -20,16 +19,3 @@ There is this compiled linux executable: - src : "import-tool", a program which writes appropriate improt trigger files for users who control the running of the import pipelines with import-tool scripts. There are numerous scripts for fetch / import / montior / notification / configuration in the "import-scripts" subdirectory. Also included are current schedule crontab entries. - -## Java Versions - -The java applications are currently compiled and run under different JAVA releases. - -### Applications maintained using Java 8 -- crdb -- darwin -- ddp - -### Applications maintained using Java 11 -- cvr -- redcap diff --git a/common/pom.xml b/common/pom.xml index 448c57bb2..068de8d1b 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -4,20 +4,15 @@ MSKCC CMO Pipelines Common Shared classes common - 0.1.0 + 1.0.0 org.mskcc.cmo.ks master - 0.1.0 + 1.0.0 - - commons-lang - commons-lang - 2.4 - javax.mail mail @@ -31,6 +26,7 @@ org.springframework.boot spring-boot-maven-plugin + ${spring.boot.version} true diff --git a/common/src/main/java/org/cbioportal/cmo/pipelines/common/util/EmailUtil.java b/common/src/main/java/org/cbioportal/cmo/pipelines/common/util/EmailUtil.java index 7d4ad02a3..7551825e7 100644 --- a/common/src/main/java/org/cbioportal/cmo/pipelines/common/util/EmailUtil.java +++ b/common/src/main/java/org/cbioportal/cmo/pipelines/common/util/EmailUtil.java @@ -32,11 +32,12 @@ package org.cbioportal.cmo.pipelines.common.util; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.*; import java.util.Properties; import javax.mail.*; import javax.mail.internet.*; -import org.apache.commons.lang.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -101,7 +102,10 @@ public void sendErrorEmail(String pipelineName, List exceptions, Stri String body = "An error occured while running the " + pipelineName + " with job parameters " + parameters + ".\n\n"; List messages = new ArrayList<>(); for (Throwable exception : exceptions) { - messages.add(ExceptionUtils.getFullStackTrace(exception)); + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + exception.printStackTrace(pw); + messages.add(pw.toString()); } body += String.join("\n\n", messages); sendEmailToDefaultRecipient(subject, body); diff --git a/crdb/pom.xml b/crdb/pom.xml index 9efefa807..27aa771df 100644 --- a/crdb/pom.xml +++ b/crdb/pom.xml @@ -4,13 +4,13 @@ MSKCC CRDB Pipeline Clinical Research Database pipeline crdb - 0.1.0 + 1.0.0 jar org.mskcc.cmo.ks master - 0.1.0 + 1.0.0 @@ -23,11 +23,6 @@ spring-web ${spring.version} - - commons-lang - commons-lang - 2.4 - commons-cli commons-cli @@ -100,7 +95,7 @@ org.mskcc.cmo.ks common - 0.1.0 + 1.0.0 jar @@ -120,6 +115,7 @@ org.springframework.boot spring-boot-maven-plugin + ${spring.boot.version} org.mskcc.cmo.ks.crdb.CRDBPipeline diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/BatchConfiguration.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/BatchConfiguration.java index aab9a9790..270c066a9 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/BatchConfiguration.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/BatchConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2019 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016 - 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -45,13 +45,16 @@ import org.mskcc.cmo.ks.crdb.pipeline.util.CRDBUtils; import org.springframework.batch.core.*; import org.springframework.batch.core.configuration.annotation.*; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.launch.support.SimpleJobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.*; import org.springframework.batch.support.transaction.ResourcelessTransactionManager; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.*; import org.springframework.core.io.Resource; @@ -67,18 +70,11 @@ */ @Configuration -@EnableBatchProcessing public class BatchConfiguration { public static final String CRDB_IMPACT_JOB = "crdbImpactJob"; public static final String CRDB_PDX_JOB = "crdbPDXJob"; - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; - @Bean public CRDBUtils crdbUtils() { return new CRDBUtils(); @@ -90,31 +86,38 @@ public EmailUtil emailUtil() { } @Bean - public Job crdbImpactJob() { - return jobBuilderFactory.get(CRDB_IMPACT_JOB) - .start(crdbSurveyStep()) - .next(crdbDatasetStep()) + public Job crdbImpactJob(JobRepository jobRepository, + @Qualifier("crdbSurveyStep") Step crdbSurveyStep, + @Qualifier("crdbDatasetStep") Step crdbDatasetStep) { + return new JobBuilder(CRDB_IMPACT_JOB, jobRepository) + .start(crdbSurveyStep) + .next(crdbDatasetStep) .build(); } @Bean - public Job crdbPDXJob() { - return jobBuilderFactory.get(CRDB_PDX_JOB) - .start(crdbPDXClinicalSampleStep()) - .next(crdbPDXClinicalPatientStep()) - .next(crdbPDXTimelineStep()) - .next(crdbPDXSourceToDestinationMappingStep()) - .next(crdbPDXClinicalAnnotationMappingStep()) + public Job crdbPDXJob(JobRepository jobRepository, + @Qualifier("crdbPDXClinicalSampleStep") Step crdbPDXClinicalSampleStep, + @Qualifier("crdbPDXClinicalPatientStep") Step crdbPDXClinicalPatientStep, + @Qualifier("crdbPDXTimelineStep") Step crdbPDXTimelineStep, + @Qualifier("crdbPDXSourceToDestinationMappingStep") Step crdbPDXSourceToDestinationMappingStep, + @Qualifier("crdbPDXClinicalAnnotationMappingStep") Step crdbPDXClinicalAnnotationMappingStep) { + return new JobBuilder(CRDB_PDX_JOB, jobRepository) + .start(crdbPDXClinicalSampleStep) + .next(crdbPDXClinicalPatientStep) + .next(crdbPDXTimelineStep) + .next(crdbPDXSourceToDestinationMappingStep) + .next(crdbPDXClinicalAnnotationMappingStep) .build(); } /** * Step 1 reads, processes, and writes the CRDB Survey query results */ - @Bean - public Step crdbSurveyStep() { - return stepBuilderFactory.get("crdbSurveyStep") - . chunk(10) + @Bean(name = "crdbSurveyStep") + public Step crdbSurveyStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbSurveyStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbSurveyReader()) .processor(crdbSurveyProcessor()) .writer(crdbSurveyWriter()) @@ -141,10 +144,10 @@ public ItemStreamWriter crdbSurveyWriter() { /** * Step 2 reads, processes, and writes the CRDB Dataset query results */ - @Bean - public Step crdbDatasetStep() { - return stepBuilderFactory.get("crdbDatasetStep") - . chunk(10) + @Bean(name = "crdbDatasetStep") + public Step crdbDatasetStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbDatasetStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbDatasetReader()) .processor(crdbDatasetProcessor()) .writer(crdbDatasetWriter()) @@ -168,10 +171,10 @@ public ItemStreamWriter crdbDatasetWriter() { return new CRDBDatasetWriter(); } - @Bean - public Step crdbPDXClinicalAnnotationMappingStep() { - return stepBuilderFactory.get("crdbPDXClinicalAnnotationMappingStep") - . chunk(10) + @Bean(name = "crdbPDXClinicalAnnotationMappingStep") + public Step crdbPDXClinicalAnnotationMappingStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbPDXClinicalAnnotationMappingStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbPDXClinicalAnnotationMappingReader()) .processor(crdbPDXClinicalAnnotationMappingProcessor()) .writer(crdbPDXClinicalAnnotationMappingWriter()) @@ -196,10 +199,10 @@ public CRDBPDXClinicalAnnotationMappingWriter crdbPDXClinicalAnnotationMappingWr return new CRDBPDXClinicalAnnotationMappingWriter(); } - @Bean - public Step crdbPDXSourceToDestinationMappingStep() { - return stepBuilderFactory.get("crdbPDXSourceToDestinationMappingStep") - . chunk(10) + @Bean(name = "crdbPDXSourceToDestinationMappingStep") + public Step crdbPDXSourceToDestinationMappingStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbPDXSourceToDestinationMappingStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbPDXSourceToDestinationMappingReader()) .processor(crdbPDXSourceToDestinationMappingProcessor()) .writer(crdbPDXSourceToDestinationMappingWriter()) @@ -224,10 +227,10 @@ public ItemStreamWriter crdbPDXSourceToDestinationMappingWriter() { return new CRDBPDXSourceToDestinationMappingWriter(); } - @Bean - public Step crdbPDXClinicalSampleStep() { - return stepBuilderFactory.get("crdbPDXClinicalSampleStep") - . chunk(10) + @Bean(name = "crdbPDXClinicalSampleStep") + public Step crdbPDXClinicalSampleStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbPDXClinicalSampleStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbPDXClinicalSampleReader()) .processor(crdbPDXClinicalSampleProcessor()) .writer(crdbPDXClinicalSampleWriter()) @@ -252,10 +255,10 @@ public ItemStreamWriter crdbPDXClinicalSampleWriter() { return new CRDBPDXClinicalSampleWriter(); } - @Bean - public Step crdbPDXClinicalPatientStep() { - return stepBuilderFactory.get("crdbPDXClinicalPatientStep") - . chunk(10) + @Bean(name = "crdbPDXClinicalPatientStep") + public Step crdbPDXClinicalPatientStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbPDXClinicalPatientStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbPDXClinicalPatientReader()) .processor(crdbPDXClinicalPatientProcessor()) .writer(crdbPDXClinicalPatientWriter()) @@ -280,10 +283,10 @@ public ItemStreamWriter crdbPDXClinicalPatientWriter() { return new CRDBPDXClinicalPatientWriter(); } - @Bean - public Step crdbPDXTimelineStep() { - return stepBuilderFactory.get("crdbPDXTimelineStep") - . chunk(10) + @Bean(name = "crdbPDXTimelineStep") + public Step crdbPDXTimelineStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("crdbPDXTimelineStep", jobRepository) + . chunk(10, transactionManager) .reader(crdbPDXTimelineReader()) .processor(crdbPDXTimelineProcessor()) .writer(crdbPDXTimelineWriter()) diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBDatasetWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBDatasetWriter.java index 948edeb62..786298dc3 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBDatasetWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBDatasetWriter.java @@ -97,11 +97,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); - for (String result : items) { - writeList.add(result); - } - flatFileItemWriter.write(writeList); + public void write(Chunk items) throws Exception { + flatFileItemWriter.write(items); } } diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalAnnotationMappingWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalAnnotationMappingWriter.java index 7457f97d8..82c38500f 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalAnnotationMappingWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalAnnotationMappingWriter.java @@ -83,7 +83,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalPatientWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalPatientWriter.java index 1eea75efd..ce00a61c2 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalPatientWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalPatientWriter.java @@ -83,7 +83,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalSampleWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalSampleWriter.java index 8a10561c2..6ced498fc 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalSampleWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXClinicalSampleWriter.java @@ -83,7 +83,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXSourceToDestinationMappingWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXSourceToDestinationMappingWriter.java index ea65e3a12..be92d21a3 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXSourceToDestinationMappingWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXSourceToDestinationMappingWriter.java @@ -83,7 +83,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXTimelineWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXTimelineWriter.java index d7f6a14e9..4491b3f2a 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXTimelineWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBPDXTimelineWriter.java @@ -86,7 +86,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } diff --git a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBSurveyWriter.java b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBSurveyWriter.java index 41fe34b8f..d632fb50c 100644 --- a/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBSurveyWriter.java +++ b/crdb/src/main/java/org/mskcc/cmo/ks/crdb/pipeline/CRDBSurveyWriter.java @@ -97,11 +97,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); - for (String result : items) { - writeList.add(result); - } - flatFileItemWriter.write(writeList); + public void write(Chunk items) throws Exception { + flatFileItemWriter.write(items); } } diff --git a/cvr/pom.xml b/cvr/pom.xml index 1963bf003..6a60c4adc 100644 --- a/cvr/pom.xml +++ b/cvr/pom.xml @@ -4,13 +4,13 @@ MSKCC CMO CVR Pipeline CVR Pipeline cvr - 0.1.0 + 1.0.0 jar org.mskcc.cmo.ks master - 0.1.0 + 1.0.0 @@ -19,11 +19,17 @@ com.github.mskcc smile-messaging-java 1.3.1.RELEASE + + + ch.qos.logback + logback-classic + + org.mskcc.cmo.ks common - 0.1.0 + 1.0.0 org.springframework @@ -35,11 +41,6 @@ jackson-databind ${jackson.version} - - commons-lang - commons-lang - 2.4 - commons-cli commons-cli @@ -68,7 +69,7 @@ com.github.genome-nexus genome-nexus-annotation-pipeline - 0f4e72e5902e309f082083068b17bffaf2a86565 + a115287d2b7699fb2149038b64f42e3a9500ff96 org.apache.logging.log4j @@ -114,6 +115,7 @@ org.springframework.boot spring-boot-maven-plugin + ${spring.boot.version} org.cbioportal.cmo.pipelines.CVRPipeline diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/BatchConfiguration.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/BatchConfiguration.java index fbeaab332..17efbda23 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/BatchConfiguration.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/BatchConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2022, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -33,43 +33,44 @@ package org.cbioportal.cmo.pipelines.cvr; import java.net.MalformedURLException; -import org.cbioportal.cmo.pipelines.cvr.model.staging.LinkedMskimpactCaseRecord; -import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeSvRecord; -import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeSegRecord; -import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeCnaRecord; -import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeClinicalRecord; -import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRSvRecord; -import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRSegRecord; -import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRGenePanelRecord; -import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRClinicalRecord; +import java.util.*; +import javax.sql.DataSource; +import org.apache.log4j.Logger; import org.cbioportal.cmo.pipelines.cvr.clinical.*; import org.cbioportal.cmo.pipelines.cvr.cna.*; import org.cbioportal.cmo.pipelines.cvr.consume.*; import org.cbioportal.cmo.pipelines.cvr.genepanel.*; import org.cbioportal.cmo.pipelines.cvr.linkedimpactcase.*; -import org.cbioportal.cmo.pipelines.cvr.samplelist.CvrSampleListsTasklet; import org.cbioportal.cmo.pipelines.cvr.model.*; +import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeClinicalRecord; +import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeCnaRecord; +import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeSegRecord; +import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeSvRecord; +import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRClinicalRecord; +import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRGenePanelRecord; +import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRSegRecord; +import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRSvRecord; +import org.cbioportal.cmo.pipelines.cvr.model.staging.LinkedMskimpactCaseRecord; import org.cbioportal.cmo.pipelines.cvr.mutation.*; import org.cbioportal.cmo.pipelines.cvr.requeue.*; +import org.cbioportal.cmo.pipelines.cvr.samplelist.CvrSampleListsTasklet; import org.cbioportal.cmo.pipelines.cvr.seg.*; +import org.cbioportal.cmo.pipelines.cvr.smile.SmilePublisherTasklet; import org.cbioportal.cmo.pipelines.cvr.sv.*; import org.cbioportal.cmo.pipelines.cvr.variants.*; import org.cbioportal.cmo.pipelines.cvr.whitelist.ZeroVariantWhitelistTasklet; import org.cbioportal.models.*; - -import java.util.*; -import javax.sql.DataSource; -import org.apache.log4j.Logger; -import org.cbioportal.cmo.pipelines.cvr.smile.SmilePublisherTasklet; import org.mskcc.cmo.messaging.Gateway; import org.springframework.batch.core.*; import org.springframework.batch.core.configuration.annotation.*; import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.*; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.launch.support.SimpleJobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.Tasklet; import org.springframework.batch.item.*; import org.springframework.batch.item.support.CompositeItemWriter; @@ -86,7 +87,6 @@ * @author heinsz */ @Configuration -@EnableBatchProcessing @ComponentScan(basePackages = {"org.cbioportal.annotator", "org.mskcc.cmo.messaging", "org.mskcc.cmo.common.*"}) public class BatchConfiguration { public static final String CVR_JOB = "cvrJob"; @@ -96,12 +96,6 @@ public class BatchConfiguration { public static final String CONSUME_SAMPLES_JOB = "consumeSamplesJob"; public static final String EMAIL_UTIL= "EmailUtil"; - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; - @Value("${chunk}") private int chunkInterval; @@ -131,326 +125,358 @@ public SvUtilities svUtilites() { } @Bean - public Job gmlJob() { - return jobBuilderFactory.get(GML_JOB) - .start(cvrSampleListsStep()) - .next(gmlJsonStep()) - .next(gmlClinicalStep()) - .next(gmlMutationStep()) - .next(gmlSvStep()) + public Job gmlJob(JobRepository jobRepository, + @Qualifier("cvrSampleListsStep") Step cvrSampleListsStep, + @Qualifier("gmlJsonStep") Step gmlJsonStep, + @Qualifier("gmlClinicalStep") Step gmlClinicalStep, + @Qualifier("gmlMutationStep") Step gmlMutationStep, + @Qualifier("gmlSvStep") Step gmlSvStep) { + return new JobBuilder(GML_JOB, jobRepository) + .start(cvrSampleListsStep) + .next(gmlJsonStep) + .next(gmlClinicalStep) + .next(gmlMutationStep) + .next(gmlSvStep) .build(); } @Bean - public Job jsonJob() { - return jobBuilderFactory.get(JSON_JOB) - .start(cvrJsonJobFlow()) + public Job jsonJob(JobRepository jobRepository, + @Qualifier("cvrJsonJobFlow") Flow cvrJsonJobFlow) { + return new JobBuilder(JSON_JOB, jobRepository) + .start(cvrJsonJobFlow) .build().build(); } @Bean - public Job cvrJob() { - return jobBuilderFactory.get(CVR_JOB) - .start(cvrResponseStep()) - .next(checkCvrReponse()) + public Job cvrJob(JobRepository jobRepository, + @Qualifier("cvrResponseStep") Step cvrResponseStep, + @Qualifier("cvrJobFlow") Flow cvrJobFlow) { + return new JobBuilder(CVR_JOB, jobRepository) + .start(cvrResponseStep) + .next(checkCvrResponse()) .on("RUN") - .to(cvrJobFlow()) - .from(checkCvrReponse()) + .to(cvrJobFlow) + .from(checkCvrResponse()) .on("STOP").end() .build().build(); } @Bean - public Job gmlJsonJob() { - return jobBuilderFactory.get(GML_JSON_JOB) - .start(cvrSampleListsStep()) - .next(gmlClinicalStep()) - .next(gmlMutationStep()) - .next(gmlSvStep()) + public Job gmlJsonJob(JobRepository jobRepository, + @Qualifier("cvrSampleListsStep") Step cvrSampleListsStep, + @Qualifier("gmlClinicalStep") Step gmlClinicalStep, + @Qualifier("gmlMutationStep") Step gmlMutationStep, + @Qualifier("gmlSvStep") Step gmlSvStep) { + return new JobBuilder(GML_JSON_JOB, jobRepository) + .start(cvrSampleListsStep) + .next(gmlClinicalStep) + .next(gmlMutationStep) + .next(gmlSvStep) .build(); } @Bean - public Job consumeSamplesJob() { - return jobBuilderFactory.get(CONSUME_SAMPLES_JOB) - .start(consumeSampleStep()) - .next(smilePublisherStep()) + public Job consumeSamplesJob(JobRepository jobRepository, + @Qualifier("consumeSampleStep") Step consumeSampleStep, + @Qualifier("smilePublisherStep") Step smilePublisherStep) { + return new JobBuilder(CONSUME_SAMPLES_JOB, jobRepository) + .start(consumeSampleStep) + .next(smilePublisherStep) .build(); } - @Bean - public Flow cvrJsonJobFlow() { + @Bean(name = "cvrJsonJobFlow") + public Flow cvrJsonJobFlow(@Qualifier("cvrSampleListsStep") Step cvrSampleListsStep, + @Qualifier("linkedMskimpactCaseFlow") Flow linkedMskimpactCaseFlow, + @Qualifier("clinicalStep") Step clinicalStep, + @Qualifier("mutationsStepFlow") Flow mutationsStepFlow, + @Qualifier("cnaStepFlow") Flow cnaStepFlow, + @Qualifier("svStepFlow") Flow svStepFlow, + @Qualifier("genePanelStep") Step genePanelStep) { return new FlowBuilder("cvrJsonJobFlow") - .start(cvrSampleListsStep()) - .next(linkedMskimpactCaseFlow()) - .next(clinicalStep()) - .next(mutationsStepFlow()) - .next(cnaStepFlow()) - .next(svStepFlow()) - .next(genePanelStep()) + .start(cvrSampleListsStep) + .next(linkedMskimpactCaseFlow) + .next(clinicalStep) + .next(mutationsStepFlow) + .next(cnaStepFlow) + .next(svStepFlow) + .next(genePanelStep) .build(); } - @Bean - public Flow cvrJobFlow() { + @Bean(name = "cvrJobFlow") + public Flow cvrJobFlow(@Qualifier("cvrSampleListsStep") Step cvrSampleListsStep, + @Qualifier("cvrJsonStep") Step cvrJsonStep, + @Qualifier("linkedMskimpactCaseFlow") Flow linkedMskimpactCaseFlow, + @Qualifier("clinicalStep") Step clinicalStep, + @Qualifier("mutationsStepFlow") Flow mutationsStepFlow, + @Qualifier("cnaStepFlow") Flow cnaStepFlow, + @Qualifier("svStepFlow") Flow svStepFlow, + @Qualifier("segmentStepFlow") Flow segmentStepFlow, + @Qualifier("genePanelStep") Step genePanelStep, + @Qualifier("cvrRequeueStep") Step cvrRequeueStep, + @Qualifier("zeroVariantWhitelistFlow") Flow zeroVariantWhitelistFlow) { return new FlowBuilder("cvrJobFlow") - .start(cvrSampleListsStep()) - .next(cvrJsonStep()) - .next(linkedMskimpactCaseFlow()) - .next(clinicalStep()) - .next(mutationsStepFlow()) - .next(cnaStepFlow()) - .next(svStepFlow()) - .next(segmentStepFlow()) - .next(genePanelStep()) - .next(cvrRequeueStep()) - .next(zeroVariantWhitelistFlow()) + .start(cvrSampleListsStep) + .next(cvrJsonStep) + .next(linkedMskimpactCaseFlow) + .next(clinicalStep) + .next(mutationsStepFlow) + .next(cnaStepFlow) + .next(svStepFlow) + .next(segmentStepFlow) + .next(genePanelStep) + .next(cvrRequeueStep) + .next(zeroVariantWhitelistFlow) .build(); } - @Bean - public Flow linkedMskimpactCaseFlow() { + @Bean(name = "linkedMskimpactCaseFlow") + public Flow linkedMskimpactCaseFlow(@Qualifier("linkedMskimpactCaseStep") Step linkedMskimpactCaseStep) { return new FlowBuilder("linkedMskimpactCaseFlow") - .start(linkedMskimpactCaseDecider()).on("RUN").to(linkedMskimpactCaseStep()) + .start(linkedMskimpactCaseDecider()).on("RUN").to(linkedMskimpactCaseStep) .from(linkedMskimpactCaseDecider()).on("SKIP").end().build(); } - @Bean - public Flow mutationsStepFlow() { + @Bean(name = "mutationsStepFlow") + public Flow mutationsStepFlow(@Qualifier("mutationStep") Step mutationStep, + @Qualifier("nonSignedoutMutationStep") Step nonSignedoutMutationStep) { return new FlowBuilder("mutationsStepFlow") .start(mutationsStepExecutionDecider()) .on("RUN") - .to(mutationStep()) - .next(nonSignedoutMutationStep()) + .to(mutationStep) + .next(nonSignedoutMutationStep) .from(mutationsStepExecutionDecider()) .on("SKIP") .end() .build(); } - @Bean - public Flow cnaStepFlow() { + @Bean(name = "cnaStepFlow") + public Flow cnaStepFlow(@Qualifier("cnaStep") Step cnaStep) { return new FlowBuilder("cnaStepFlow") .start(cnaStepExecutionDecider()) .on("RUN") - .to(cnaStep()) + .to(cnaStep) .from(cnaStepExecutionDecider()) .on("SKIP") .end() .build(); } - @Bean - public Flow svStepFlow() { + @Bean(name = "svStepFlow") + public Flow svStepFlow(@Qualifier("svStep") Step svStep) { return new FlowBuilder("svStepFlow") .start(svStepExecutionDecider()) .on("RUN") - .to(svStep()) + .to(svStep) .from(svStepExecutionDecider()) .on("SKIP") .end() .build(); } - @Bean - public Flow segmentStepFlow() { + @Bean(name = "segmentStepFlow") + public Flow segmentStepFlow(@Qualifier("segStep") Step segStep) { return new FlowBuilder("segmentStepFlow") .start(segStepExecutionDecider()) .on("RUN") - .to(segStep()) + .to(segStep) .from(segStepExecutionDecider()) .on("SKIP") .end() .build(); } - @Bean - public Flow zeroVariantWhitelistFlow() { + @Bean(name = "zeroVariantWhitelistFlow") + public Flow zeroVariantWhitelistFlow(@Qualifier("zeroVariantWhitelistStep") Step zeroVariantWhitelistStep) { // use datatype = mutations since whitelist is for mutations data return new FlowBuilder("zeroVariantWhitelistFlow") .start(mutationsStepExecutionDecider()) .on("RUN") - .to(zeroVariantWhitelistStep()) + .to(zeroVariantWhitelistStep) .from(mutationsStepExecutionDecider()) .on("SKIP") .end() .build(); } - @Bean - public Step gmlJsonStep() { - return stepBuilderFactory.get("gmlJsonStep") - . chunk(chunkInterval) + @Bean(name = "gmlJsonStep") + public Step gmlJsonStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("gmlJsonStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(gmlJsonReader()) .processor(gmlJsonProcessor()) .writer(gmlJsonWriter()) .build(); } - @Bean - public Step gmlMutationStep() { - return stepBuilderFactory.get("gmlMutationStep") - . chunk(chunkInterval) + @Bean(name = "gmlMutationStep") + public Step gmlMutationStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("gmlMutationStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(gmlMutationReader()) .processor(mutationDataProcessor()) .writer(mutationDataWriter()) .build(); } - @Bean - public Step gmlClinicalStep() { - return stepBuilderFactory.get("gmlClinicalStep") - .tasklet(gmlClinicalTasklet()) + @Bean(name = "gmlClinicalStep") + public Step gmlClinicalStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, @Qualifier("gmlClinicalTasklet") Tasklet gmlClinicalTasklet) { + return new StepBuilder("gmlClinicalStep", jobRepository) + .tasklet(gmlClinicalTasklet, transactionManager) .build(); } - @Bean + @Bean(name = "gmlClinicalTasklet") @StepScope public Tasklet gmlClinicalTasklet() { return new GMLClinicalTasklet(); } - @Bean - public Step gmlSvStep() { - return stepBuilderFactory.get("gmlSvStep") - . chunk(chunkInterval) + @Bean(name = "gmlSvStep") + public Step gmlSvStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("gmlSvStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(gmlSvDataReader()) .processor(svDataProcessor()) .writer(svDataWriter()) .build(); } - @Bean - public Step cvrJsonStep() { - return stepBuilderFactory.get("cvrJsonStep") + @Bean(name = "cvrJsonStep") + public Step cvrJsonStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("cvrJsonStep", jobRepository) .listener(cvrResponseListener()) - . chunk(chunkInterval) - .reader(cvrJsonreader()) - .processor(cvrJsonprocessor()) - .writer(cvrJsonwriter()) + . chunk(chunkInterval, transactionManager) + .reader(cvrJsonReader()) + .processor(cvrJsonProcessor()) + .writer(cvrJsonWriter()) .build(); } - @Bean - public Step clinicalStep() { - return stepBuilderFactory.get("clinicalStep") - . chunk(chunkInterval) + @Bean(name = "clinicalStep") + public Step clinicalStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("clinicalStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(clinicalDataReader()) .processor(clinicalDataProcessor()) .writer(compositeClinicalDataWriter()) .build(); } - @Bean - public Step linkedMskimpactCaseStep() { - return stepBuilderFactory.get("linkedMskimpactCaseStep") - . chunk(chunkInterval) + @Bean(name = "linkedMskimpactCaseStep") + public Step linkedMskimpactCaseStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("linkedMskimpactCaseStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(linkedMskimpactCaseReader()) .processor(linkedMskimpactCaseProcessor()) .writer(linkedMskimpactCaseWriter()) .build(); } - @Bean - public Step mutationStep() { - return stepBuilderFactory.get("mutationStep") - . chunk(chunkInterval) + @Bean(name = "mutationStep") + public Step mutationStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("mutationStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(mutationDataReader()) .processor(mutationDataProcessor()) .writer(mutationDataWriter()) .build(); } - @Bean - public Step nonSignedoutMutationStep() { - return stepBuilderFactory.get("nonSignedoutMutationStep") - . chunk(chunkInterval) + @Bean(name = "nonSignedoutMutationStep") + public Step nonSignedoutMutationStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("nonSignedoutMutationStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(nonSignedoutMutationDataReader()) .processor(mutationDataProcessor()) .writer(mutationDataWriter()) .build(); } - @Bean - public Step cnaStep() { - return stepBuilderFactory.get("cnaStep") - . chunk(chunkInterval) + @Bean(name = "cnaStep") + public Step cnaStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("cnaStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(cnaDataReader()) .writer(cnaDataWriter()) .build(); } - @Bean - public Step svStep() { - return stepBuilderFactory.get("svStep") - . chunk(chunkInterval) + @Bean(name = "svStep") + public Step svStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("svStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(svDataReader()) .processor(svDataProcessor()) .writer(svDataWriter()) .build(); } - @Bean - public Step segStep() { - return stepBuilderFactory.get("segStep") - . chunk(chunkInterval) + @Bean(name = "segStep") + public Step segStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("segStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(segDataReader()) .processor(segDataProcessor()) .writer(segDataWriter()) .build(); } - @Bean - public Step genePanelStep() { - return stepBuilderFactory.get("genePanelStep"). chunk(chunkInterval) + @Bean(name = "genePanelStep") + public Step genePanelStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("genePanelStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(genePanelReader()) .processor(genePanelProcessor()) .writer(genePanelWriter()) .build(); } - @Bean - public Step cvrRequeueStep() { - return stepBuilderFactory.get("cvrRequeueStep") + @Bean(name = "cvrRequeueStep") + public Step cvrRequeueStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, @Qualifier("cvrRequeueTasklet") Tasklet cvrRequeueTasklet) { + return new StepBuilder("cvrRequeueStep", jobRepository) .listener(cvrRequeueListener()) - .tasklet(cvrRequeueTasklet()) + .tasklet(cvrRequeueTasklet, transactionManager) .build(); } - @Bean - public Step smilePublisherStep() { - return stepBuilderFactory.get("smilePublisherStep") - .tasklet(smilePublisherTasklet()) + @Bean(name = "smilePublisherStep") + public Step smilePublisherStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, @Qualifier("smilePublisherTasklet") Tasklet smilePublisherTasklet) { + return new StepBuilder("smilePublisherStep", jobRepository) + .tasklet(smilePublisherTasklet, transactionManager) .build(); } - @Bean - public Step consumeSampleStep() { - return stepBuilderFactory.get("consumeSampleStep") - . chunk(chunkInterval) + @Bean(name = "consumeSampleStep") + public Step consumeSampleStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("consumeSampleStep", jobRepository) + . chunk(chunkInterval, transactionManager) .reader(consumeSampleReader()) .writer(consumeSampleWriter()) .build(); } - @Bean - public Step cvrSampleListsStep() { - return stepBuilderFactory.get("cvrSampleListsStep") - .tasklet(cvrSampleListsTasklet()) + @Bean(name = "cvrSampleListsStep") + public Step cvrSampleListsStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, @Qualifier("cvrSampleListsTasklet") Tasklet cvrSampleListsTasklet) { + return new StepBuilder("cvrSampleListsStep", jobRepository) + .tasklet(cvrSampleListsTasklet, transactionManager) .build(); } - @Bean - public Step cvrResponseStep() { - return stepBuilderFactory.get("cvrResponseStep") - .tasklet(cvrResponseTasklet()) + @Bean(name = "cvrResponseStep") + public Step cvrResponseStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, @Qualifier("cvrResponseTasklet") Tasklet cvrResponseTasklet) { + return new StepBuilder("cvrResponseStep", jobRepository) + .tasklet(cvrResponseTasklet, transactionManager) .build(); } - @Bean - public Step zeroVariantWhitelistStep() { - return stepBuilderFactory.get("zeroVariantWhitelistStep") - .tasklet(zeroVariantWhitelistTasklet()) + @Bean(name = "zeroVariantWhitelistStep") + public Step zeroVariantWhitelistStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, @Qualifier("zeroVariantWhitelistTasklet") Tasklet zeroVariantWhitelistTasklet) { + return new StepBuilder("zeroVariantWhitelistStep", jobRepository) + .tasklet(zeroVariantWhitelistTasklet, transactionManager) .build(); } @@ -482,21 +508,21 @@ public ItemStreamReader gmlMutationReader() { // Reader to get json data from CVR @Bean @StepScope - public ItemStreamReader cvrJsonreader() { + public ItemStreamReader cvrJsonReader() { return new CVRVariantsReader(); } // Processor for processing the json data from CVR @Bean @StepScope - public CVRVariantsProcessor cvrJsonprocessor() { + public CVRVariantsProcessor cvrJsonProcessor() { return new CVRVariantsProcessor(); } // Writer for writing out json from CVR to file @Bean @StepScope - public ItemStreamWriter cvrJsonwriter() { + public ItemStreamWriter cvrJsonWriter() { return new CVRVariantsWriter(); } @@ -661,13 +687,13 @@ public ItemStreamWriter genePanelWriter() { return new CVRGenePanelWriter(); } - @Bean + @Bean(name = "cvrSampleListsTasklet") @StepScope public Tasklet cvrSampleListsTasklet() { return new CvrSampleListsTasklet(); } - @Bean + @Bean(name = "cvrRequeueTasklet") @StepScope public Tasklet cvrRequeueTasklet() { return new CvrRequeueTasklet(); @@ -695,7 +721,7 @@ public ItemStreamWriter consumeSampleWriter() { return new ConsumeSampleWriter(); } - @Bean + @Bean(name = "smilePublisherTasklet") @StepScope public Tasklet smilePublisherTasklet() { return new SmilePublisherTasklet(); @@ -717,20 +743,20 @@ public FlowExecutionStatus decide(JobExecution je, StepExecution se) { }; } - @Bean + @Bean(name = "cvrResponseTasklet") @StepScope public Tasklet cvrResponseTasklet() { return new CvrResponseTasklet(); } - @Bean + @Bean(name = "zeroVariantWhitelistTasklet") @StepScope public Tasklet zeroVariantWhitelistTasklet() { return new ZeroVariantWhitelistTasklet(); } @Bean - public JobExecutionDecider checkCvrReponse() { + public JobExecutionDecider checkCvrResponse() { return new JobExecutionDecider() { @Override public FlowExecutionStatus decide(JobExecution je, StepExecution se) { diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/CVRUtilities.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/CVRUtilities.java index e211a0007..8f0cf2d56 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/CVRUtilities.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/CVRUtilities.java @@ -177,12 +177,12 @@ public List processFileComments(File dataFile) throws FileNotFoundExcept return comments; } - public boolean isDuplicateRecord(MutationRecord snp, List annotatedRecords) { - if (annotatedRecords == null || annotatedRecords.isEmpty()) { + public boolean isDuplicateRecord(MutationRecord snp, List mutationRecords) { + if (mutationRecords == null || mutationRecords.isEmpty()) { return false; } - for (AnnotatedRecord record : annotatedRecords) { + for (MutationRecord record : mutationRecords) { if (record.getCHROMOSOME().equals(snp.getCHROMOSOME()) && record.getSTART_POSITION().equals(snp.getSTART_POSITION()) && record.getEND_POSITION().equals(snp.getEND_POSITION()) && diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRClinicalDataWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRClinicalDataWriter.java index 3d0b381ac..fa9ab29a0 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRClinicalDataWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRClinicalDataWriter.java @@ -87,8 +87,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); + public void write(Chunk items) throws Exception { + Chunk writeList = new Chunk<>(); for (CompositeClinicalRecord item : items) { if (item.getNewClinicalRecord() != null) { writeList.add(item.getNewClinicalRecord()); diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRSeqDateClinicalDataWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRSeqDateClinicalDataWriter.java index 8a32d66ea..7fbd1229f 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRSeqDateClinicalDataWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/clinical/CVRSeqDateClinicalDataWriter.java @@ -91,9 +91,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { if (CVRUtilities.SUPPORTED_SEQ_DATE_STUDY_IDS.contains(studyId)) { - List writeList = new ArrayList<>(); + Chunk writeList = new Chunk<>(); for (CompositeClinicalRecord item : items) { if (item.getSeqDateRecord()!= null) { writeList.add(item.getSeqDateRecord()); diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/cna/CVRCnaDataWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/cna/CVRCnaDataWriter.java index b07d410e6..6592c2b47 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/cna/CVRCnaDataWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/cna/CVRCnaDataWriter.java @@ -33,10 +33,9 @@ package org.cbioportal.cmo.pipelines.cvr.cna; import java.io.File; -import java.util.ArrayList; -import java.util.List; import org.cbioportal.cmo.pipelines.cvr.CVRUtilities; import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeCnaRecord; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -80,8 +79,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List records) throws Exception { - List items = new ArrayList<>(); + public void write(Chunk records) throws Exception { + Chunk items = new Chunk<>(); for (CompositeCnaRecord record : records) { if (record.getAllCnaRecord() != null) { items.add(record.getAllCnaRecord()); diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/consume/ConsumeSampleWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/consume/ConsumeSampleWriter.java index 25b63178c..1d33c000e 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/consume/ConsumeSampleWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/consume/ConsumeSampleWriter.java @@ -33,13 +33,13 @@ package org.cbioportal.cmo.pipelines.cvr.consume; import java.time.Instant; -import java.util.List; import org.apache.log4j.Logger; import org.cbioportal.cmo.pipelines.common.util.HttpClientWithTimeoutAndRetry; import org.cbioportal.cmo.pipelines.common.util.InstantStringUtil; import org.cbioportal.cmo.pipelines.cvr.CvrSampleListUtil; import org.cbioportal.cmo.pipelines.cvr.CVRUtilities; import org.cbioportal.cmo.pipelines.cvr.model.CVRConsumeSample; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -116,7 +116,7 @@ public void update(ExecutionContext ec) throws ItemStreamException {} public void close() throws ItemStreamException {} @Override - public void write(List sampleIdList) throws Exception { + public void write(Chunk sampleIdList) throws Exception { for (String sampleId : sampleIdList) { if (testingMode) { log.info("[TESTING MODE]: sample id will not be consumed: " + sampleId); diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/genepanel/CVRGenePanelWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/genepanel/CVRGenePanelWriter.java index 5749777d0..b22dd0207 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/genepanel/CVRGenePanelWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/genepanel/CVRGenePanelWriter.java @@ -85,7 +85,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/linkedimpactcase/LinkedMskimpactCaseWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/linkedimpactcase/LinkedMskimpactCaseWriter.java index 753dc87a5..6d34e26b8 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/linkedimpactcase/LinkedMskimpactCaseWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/linkedimpactcase/LinkedMskimpactCaseWriter.java @@ -35,12 +35,11 @@ import java.io.File; import java.io.IOException; import java.io.Writer; -import java.util.ArrayList; -import java.util.List; import org.apache.log4j.Logger; import org.cbioportal.cmo.pipelines.cvr.CVRUtilities; import org.cbioportal.cmo.pipelines.cvr.model.*; import org.cbioportal.cmo.pipelines.cvr.model.staging.LinkedMskimpactCaseRecord; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.file.FlatFileHeaderCallback; import org.springframework.batch.item.file.FlatFileItemWriter; @@ -88,7 +87,7 @@ public void update(ExecutionContext ec) throws ItemStreamException {} public void close() throws ItemStreamException {} @Override - public void write(List list) throws Exception { + public void write(Chunk list) throws Exception { flatFileItemWriter.write(list); } } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvIntragenicVariant.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvIntragenicVariant.java index 5895a9ac0..43367a06b 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvIntragenicVariant.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvIntragenicVariant.java @@ -46,8 +46,7 @@ import java.util.*; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -296,11 +295,6 @@ public void setVariantStatusCvId(Integer variantStatusCvId) { this.variantStatusCvId = variantStatusCvId; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvVariant.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvVariant.java index 4e85acc67..919036d4e 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvVariant.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRCnvVariant.java @@ -45,7 +45,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRData.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRData.java index 8f2f1ac1d..009b870ef 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRData.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRData.java @@ -43,8 +43,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.*; -import javax.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMasterList.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMasterList.java index 8163a7f99..3f6b3176c 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMasterList.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMasterList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -46,8 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -143,11 +142,6 @@ public void setSampleCount(Integer sampleCount) { this.sampleCount = sampleCount; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMergedResult.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMergedResult.java index b797a2509..01be91795 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMergedResult.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMergedResult.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -43,9 +43,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.*; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -311,11 +310,6 @@ public void setSvVariants(List svVariants) { this.svVariants = svVariants; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMetaData.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMetaData.java index 5f35397ce..3c08b55e1 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMetaData.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRMetaData.java @@ -46,7 +46,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.google.common.base.Strings; import java.util.*; -import javax.annotation.Generated; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRRequeueRecord.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRRequeueRecord.java index 5e0056f8b..70d0abaf1 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRRequeueRecord.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRRequeueRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -45,8 +45,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -166,11 +165,6 @@ public void setSampleId(String sampleId) { this.sampleId = sampleId; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRResult.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRResult.java index 0764700ad..863f82d8d 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRResult.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRResult.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -43,9 +43,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.*; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -273,11 +272,6 @@ public void setSvVariants(List svVariants) { this.svVariants = svVariants; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSegData.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSegData.java index 5e7426bac..d54bc8101 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSegData.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSegData.java @@ -42,9 +42,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.annotation.Generated; import java.util.*; import java.util.HashMap; -import javax.annotation.Generated; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSession.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSession.java index b5d9de5c4..7f53b05d3 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSession.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSession.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -45,8 +45,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -169,11 +168,6 @@ public void setDisclaimer(String disclaimer) { this.disclaimer = disclaimer; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSnp.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSnp.java index 010ae4864..7caaf0643 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSnp.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSnp.java @@ -44,9 +44,9 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSvVariant.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSvVariant.java index 0296d2da1..9f004aa4d 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSvVariant.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CVRSvVariant.java @@ -43,9 +43,9 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CvrResponse.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CvrResponse.java index 667c2e619..ee1e26143 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CvrResponse.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/CvrResponse.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -33,9 +33,8 @@ package org.cbioportal.cmo.pipelines.cvr.model; import com.fasterxml.jackson.annotation.*; +import jakarta.annotation.Generated; import java.util.*; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -154,11 +153,6 @@ public void setInformation(String information) { this.information = information; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLCnvIntragenicVariant.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLCnvIntragenicVariant.java index b7e9ef2b7..ab367763d 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLCnvIntragenicVariant.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLCnvIntragenicVariant.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -46,8 +46,7 @@ import java.util.*; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLData.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLData.java index cd62b3dea..f74df6756 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLData.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLData.java @@ -44,7 +44,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.*; -import javax.annotation.Generated; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLMetaData.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLMetaData.java index cc874a82f..c603279b5 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLMetaData.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLMetaData.java @@ -44,7 +44,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.*; -import javax.annotation.Generated; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLResult.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLResult.java index 086d0bc74..2d4f8b64f 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLResult.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLResult.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -43,9 +43,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.*; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -120,11 +119,6 @@ public void setSnpIndelGml(List snpIndelGml) { this.snpIndelGml = snpIndelGml; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLSnp.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLSnp.java index d45d958c8..3305bc214 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLSnp.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLSnp.java @@ -40,7 +40,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; +import jakarta.annotation.Generated; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLVariant.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLVariant.java index 6295db16e..a0312f995 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLVariant.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/model/GMLVariant.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2017, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -43,10 +43,9 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.annotation.Generated; import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; -import org.apache.commons.lang.builder.ToStringBuilder; @JsonInclude(JsonInclude.Include.NON_NULL) @Generated("org.jsonschema2pojo") @@ -144,11 +143,6 @@ public void setSampleCount(Integer sampleCount) { this.sampleCount = sampleCount; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - @JsonAnyGetter public Map getAdditionalProperties() { return this.additionalProperties; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataReader.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataReader.java index bef3efeb7..3242aab00 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataReader.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataReader.java @@ -80,7 +80,7 @@ public class CVRMutationDataReader implements ItemStreamReader private Annotator annotator; private List mutationRecords = new ArrayList<>(); - private Map> mutationMap = new HashMap<>(); + private Map> mutationMap = new HashMap<>(); private File mutationFile; Set header = new LinkedHashSet<>(); @@ -132,7 +132,9 @@ private void loadMutationRecordsFromJson(CVRData cvrData) { String somaticStatus = result.getMetaData().getSomaticStatus() != null ? result.getMetaData().getSomaticStatus() : "N/A"; int countSignedOutSnps = result.getAllSignedoutCvrSnps().size(); for (CVRSnp snp : result.getAllSignedoutCvrSnps()) { - recordsToAnnotate.add(cvrUtilities.buildCVRMutationRecord(snp, sampleId, somaticStatus)); + MutationRecord to_add = cvrUtilities.buildCVRMutationRecord(snp, sampleId, somaticStatus); + recordsToAnnotate.add(to_add); + addRecordToMap(to_add); } cvrSampleListUtil.updateSignedoutSampleSnpCounts(sampleId, countSignedOutSnps); if (!stopZeroVariantWarnings && countSignedOutSnps == 0) { @@ -176,6 +178,7 @@ public void handleLine(String line) { } cvrSampleListUtil.updateSignedoutSampleSnpCounts(to_add.getTUMOR_SAMPLE_BARCODE(), 1); recordsToAnnotate.add(to_add); + addRecordToMap(to_add); } reader.close(); log.info("Loaded " + String.valueOf(recordsToAnnotate.size()) + " records from MAF"); @@ -192,7 +195,6 @@ private void annotateRecordsWithPOST(List records, boolean reann for (AnnotatedRecord ar : annotatedRecords) { logAnnotationProgress(++annotatedVariantsCount, totalVariantsToAnnotateCount, postIntervalSize); mutationRecords.add(ar); - mutationMap.getOrDefault(ar.getTUMOR_SAMPLE_BARCODE(), new ArrayList()).add(ar); additionalPropertyKeys.addAll(ar.getAdditionalProperties().keySet()); header.addAll(ar.getHeaderWithAdditionalFields()); } @@ -231,4 +233,16 @@ public AnnotatedRecord read() throws Exception { } return null; } + + private void addRecordToMap(MutationRecord record) { + String sampleId = record.getTUMOR_SAMPLE_BARCODE(); + List recordList = mutationMap.get(sampleId); + if (recordList == null) { + recordList = new ArrayList(); + recordList.add(record); + mutationMap.put(sampleId, recordList); + } else { + recordList.add(record); + } + } } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataWriter.java index 80e74ba83..9c44cd6b4 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRMutationDataWriter.java @@ -106,7 +106,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRNonSignedoutMutationDataReader.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRNonSignedoutMutationDataReader.java index 7cfb68d59..7f7722406 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRNonSignedoutMutationDataReader.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/CVRNonSignedoutMutationDataReader.java @@ -77,7 +77,7 @@ public class CVRNonSignedoutMutationDataReader implements ItemStreamReader mutationRecords = new ArrayList<>(); - private Map> mutationMap = new HashMap<>(); + private Map> mutationMap = new HashMap<>(); private Set additionalPropertyKeys = new LinkedHashSet<>(); Set header = new LinkedHashSet<>(); private AnnotationSummaryStatistics summaryStatistics; @@ -128,7 +128,9 @@ private void loadMutationRecordsFromJson(CVRData cvrData) { int countNonSignedoutSampleSnps = result.getAllNonSignedoutCvrSnps().size(); String somaticStatus = result.getMetaData().getSomaticStatus() != null ? result.getMetaData().getSomaticStatus() : "N/A"; for (CVRSnp snp : result.getAllNonSignedoutCvrSnps()) { - recordsToAnnotate.add(cvrUtilities.buildCVRMutationRecord(snp, sampleId, somaticStatus)); + MutationRecord to_add = cvrUtilities.buildCVRMutationRecord(snp, sampleId, somaticStatus); + recordsToAnnotate.add(to_add); + addRecordToMap(to_add); } cvrSampleListUtil.updateNonSignedoutSampleSnpCount(sampleId, countNonSignedoutSampleSnps); } @@ -169,6 +171,7 @@ public void handleLine(String line) { } cvrSampleListUtil.updateNonSignedoutSampleSnpCount(to_add.getTUMOR_SAMPLE_BARCODE(), 1); recordsToAnnotate.add(to_add); + addRecordToMap(to_add); } reader.close(); log.info("Loaded " + String.valueOf(recordsToAnnotate.size()) + " records from MAF"); @@ -185,7 +188,6 @@ private void annotateRecordsWithPOST(List records, boolean reann for (AnnotatedRecord ar : annotatedRecords) { logAnnotationProgress(++annotatedVariantsCount, totalVariantsToAnnotateCount, postIntervalSize); mutationRecords.add(ar); - mutationMap.getOrDefault(ar.getTUMOR_SAMPLE_BARCODE(), new ArrayList()).add(ar); additionalPropertyKeys.addAll(ar.getAdditionalProperties().keySet()); header.addAll(ar.getHeaderWithAdditionalFields()); } @@ -224,4 +226,17 @@ public AnnotatedRecord read() throws Exception { } return null; } + + private void addRecordToMap(MutationRecord record) { + String sampleId = record.getTUMOR_SAMPLE_BARCODE(); + List recordList = mutationMap.get(sampleId); + if (recordList == null) { + recordList = new ArrayList(); + recordList.add(record); + mutationMap.put(sampleId, recordList); + } else { + recordList.add(record); + } + } + } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/GMLMutationDataReader.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/GMLMutationDataReader.java index c7a80b4ac..8dbf27315 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/GMLMutationDataReader.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/mutation/GMLMutationDataReader.java @@ -76,8 +76,8 @@ public class GMLMutationDataReader implements ItemStreamReader @Autowired private Annotator annotator; - private List mutationRecords = new ArrayList(); - private Map> mutationMap = new HashMap<>(); + private List mutationRecords = new ArrayList<>(); + private Map> mutationMap = new HashMap<>(); private File mutationFile; private Set additionalPropertyKeys = new LinkedHashSet<>(); private Set header = new LinkedHashSet<>(); @@ -132,8 +132,10 @@ private void loadMutationRecordsFromJson(GMLData gmlData) { if (samples != null && !snps.isEmpty()) { for (GMLSnp snp : snps) { for (String sampleId : samples) { - recordsToAnnotate.add(cvrUtilities.buildGMLMutationRecord(snp, sampleId)); + MutationRecord to_add = cvrUtilities.buildGMLMutationRecord(snp, sampleId); + recordsToAnnotate.add(to_add); germlineSamples.add(sampleId); + addRecordToMap(to_add); } } } @@ -174,6 +176,7 @@ public void handleLine(String line) { continue; } recordsToAnnotate.add(to_add); + addRecordToMap(to_add); } reader.close(); log.info("Loaded " + String.valueOf(recordsToAnnotate.size()) + " records from MAF"); @@ -190,7 +193,6 @@ private List annotateRecordsWithPOST(List recor for (AnnotatedRecord ar : annotatedRecords) { logAnnotationProgress(++annotatedVariantsCount, totalVariantsToAnnotateCount, postIntervalSize); mutationRecords.add(ar); - mutationMap.getOrDefault(ar.getTUMOR_SAMPLE_BARCODE(), new ArrayList()).add(ar); additionalPropertyKeys.addAll(ar.getAdditionalProperties().keySet()); header.addAll(ar.getHeaderWithAdditionalFields()); } @@ -231,4 +233,15 @@ public AnnotatedRecord read() throws Exception { return null; } + private void addRecordToMap(MutationRecord record) { + String sampleId = record.getTUMOR_SAMPLE_BARCODE(); + List recordList = mutationMap.get(sampleId); + if (recordList == null) { + recordList = new ArrayList(); + recordList.add(record); + mutationMap.put(sampleId, recordList); + } else { + recordList.add(record); + } + } } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/samplelist/CvrSampleListsTasklet.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/samplelist/CvrSampleListsTasklet.java index e37eb4878..7e9eee32f 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/samplelist/CvrSampleListsTasklet.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/samplelist/CvrSampleListsTasklet.java @@ -32,10 +32,10 @@ package org.cbioportal.cmo.pipelines.cvr.samplelist; +import jakarta.annotation.Resource; import java.io.*; import java.time.Instant; import java.util.*; -import javax.annotation.Resource; import org.apache.log4j.Logger; import org.cbioportal.cmo.pipelines.common.util.HttpClientWithTimeoutAndRetry; import org.cbioportal.cmo.pipelines.common.util.InstantStringUtil; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/seg/CVRSegDataWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/seg/CVRSegDataWriter.java index 91e329d26..3f7e794fb 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/seg/CVRSegDataWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/seg/CVRSegDataWriter.java @@ -87,8 +87,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); + public void write(Chunk items) throws Exception { + Chunk writeList = new Chunk<>(); for (CompositeSegRecord item : items) { if (item.getNewSegRecord() != null) { writeList.add(item.getNewSegRecord()); diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/sv/CVRSvDataWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/sv/CVRSvDataWriter.java index e8dc2e1d5..1b4c5cad4 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/sv/CVRSvDataWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/sv/CVRSvDataWriter.java @@ -39,6 +39,7 @@ import org.cbioportal.cmo.pipelines.cvr.CVRUtilities; import org.cbioportal.cmo.pipelines.cvr.model.composite.CompositeSvRecord; import org.cbioportal.cmo.pipelines.cvr.model.staging.CVRSvRecord; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.file.FlatFileHeaderCallback; import org.springframework.batch.item.file.FlatFileItemWriter; @@ -89,8 +90,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); + public void write(Chunk items) throws Exception { + Chunk writeList = new Chunk<>(); for (CompositeSvRecord item : items) { if (item.getNewSvRecord() != null) { writeList.add(item.getNewSvRecord()); diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CVRVariantsWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CVRVariantsWriter.java index 7464dc3e6..cf0051f8b 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CVRVariantsWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CVRVariantsWriter.java @@ -80,7 +80,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CvrResponseTasklet.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CvrResponseTasklet.java index 4fa9d1e83..5f8689afe 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CvrResponseTasklet.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/CvrResponseTasklet.java @@ -34,7 +34,7 @@ import java.time.Instant; import java.util.Map; -import javax.annotation.Resource; +import jakarta.annotation.Resource; import org.apache.log4j.Logger; import org.cbioportal.cmo.pipelines.common.util.HttpClientWithTimeoutAndRetry; import org.cbioportal.cmo.pipelines.common.util.InstantStringUtil; diff --git a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/GMLVariantsWriter.java b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/GMLVariantsWriter.java index cd3c56b58..08e0e8d59 100644 --- a/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/GMLVariantsWriter.java +++ b/cvr/src/main/java/org/cbioportal/cmo/pipelines/cvr/variants/GMLVariantsWriter.java @@ -33,7 +33,6 @@ package org.cbioportal.cmo.pipelines.cvr.variants; import java.io.File; -import java.util.List; import org.cbioportal.cmo.pipelines.cvr.CVRUtilities; import org.springframework.batch.item.*; import org.springframework.batch.item.file.*; @@ -75,7 +74,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { flatFileItemWriter.write(items); } } diff --git a/darwin/pom.xml b/darwin/pom.xml deleted file mode 100644 index f31ddcae3..000000000 --- a/darwin/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - 4.0.0 - MSKCC CMO Darwin Pipeline - darwin - Darwin database pipeline - 0.1.0 - org.mskcc.cmo.ks.darwin - - jar - - org.mskcc.cmo.ks - master - 0.1.0 - - - - - org.mskcc.cmo.ks - common - 0.1.0 - - - javax.validation - validation-api - 1.1.0.Final - - - commons-collections - commons-collections - 3.2.2 - - - org.springframework - spring-web - ${spring.version} - jar - - - com.mysema.querydsl - querydsl-sql - 3.7.2 - jar - - - com.querydsl - querydsl-sql-spatial - 4.0.9 - jar - - - com.querydsl - querydsl-maven-plugin - 4.1.1 - - - org.hibernate - hibernate-validator-annotation-processor - 5.1.3.Final - - - javax.el - javax.el-api - 2.2.4 - - - org.glassfish.web - javax.el - 2.2.4 - - - cglib - cglib-nodep - 3.2.2 - - - com.google.guava - guava - 19.0 - - - - com.ibm.db2.jcc - db2jcc - db2jcc4 - - - org.mskcc.cmo.ks.redcap.source - redcap_source - 0.1.0 - jar - - - - - darwin_fetcher - - - - org.springframework.boot - spring-boot-maven-plugin - - org.mskcc.cmo.ks.darwin.DarwinPipeline - - - - - - diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/DarwinPipeline.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/DarwinPipeline.java deleted file mode 100644 index 89cba7f78..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/DarwinPipeline.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin; - -import org.mskcc.cmo.ks.darwin.pipeline.BatchConfiguration; -import org.apache.commons.cli.*; -import org.apache.commons.cli.Options; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.batch.core.*; -import org.springframework.batch.core.launch.JobLauncher; -import org.apache.log4j.Logger; -import org.springframework.boot.WebApplicationType; -/** - * - * @author jake - */ -@SpringBootApplication -public class DarwinPipeline { - - private static Logger log = Logger.getLogger(DarwinPipeline.class); - - private static Options getOptions(String[] args){ - Options options = new Options(); - options.addOption("h", "help", false, "shows this help document and quits.") - .addOption("d", "directory", true, "Output directory") - .addOption("s", "study", true, "Cancer Study ID") - .addOption("r", "current_demographics_rec_count", false, "Count of records in current demographics file. Used to sanity check num of records in latest demographics data fetch. Required only when not running the CAISIS only mode.") - .addOption("c", "caisis_mode", false, "Flag to run the MSK CAISIS job only"); - return options; - } - - private static void help(Options options, int exitStatus){ - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.printHelp("Darwin Pipeline", options); - System.exit(exitStatus); - } - - private static void launchJob(String[] args, String outputDirectory, String studyID, String currentDemographicsRecCount) throws Exception{ - SpringApplication app = new SpringApplication(DarwinPipeline.class); - app.setWebApplicationType(WebApplicationType.NONE); - ConfigurableApplicationContext ctx = app.run(args); - JobLauncher jobLauncher = ctx.getBean(JobLauncher.class); - - String jobName; - JobParameters jobParameters = new JobParametersBuilder() - .addString("outputDirectory", outputDirectory) - .addString("studyID", studyID) - .addString("currentDemographicsRecCount", currentDemographicsRecCount) - .toJobParameters(); - switch (studyID) { - case "mskimpact": - jobName = BatchConfiguration.MSKIMPACT_JOB; - break; - case "skcm_mskcc_2015_chant": - jobName = BatchConfiguration.SKCM_MSKCC_2015_CHANT_JOB; - break; - default: - jobName = BatchConfiguration.MSK_JOB; - break; - } - Job job = ctx.getBean(jobName, Job.class); - JobExecution jobExecution = jobLauncher.run(job, jobParameters); - if (!jobExecution.getExitStatus().equals(ExitStatus.COMPLETED)) { - log.error("DarwinPipeline Job '" + jobName + "' exited with status: " + jobExecution.getExitStatus()); - System.exit(2); - } - } - - private static void launchCaisisJob(String[] args, String outputDirectory, String studyID) throws Exception { - SpringApplication app = new SpringApplication(DarwinPipeline.class); - app.setWebApplicationType(WebApplicationType.NONE); - ConfigurableApplicationContext ctx = app.run(args); - JobLauncher jobLauncher = ctx.getBean(JobLauncher.class); - - JobParameters jobParameters = new JobParametersBuilder() - .addString("outputDirectory", outputDirectory) - .addString("studyID", studyID) - .toJobParameters(); - Job job = ctx.getBean(BatchConfiguration.MSK_CAISIS_JOB, Job.class); - JobExecution jobExecution = jobLauncher.run(job, jobParameters); - if (!jobExecution.getExitStatus().equals(ExitStatus.COMPLETED)) { - log.error("DarwinPipeline Job '" + BatchConfiguration.MSK_CAISIS_JOB + "' exited with status: " + jobExecution.getExitStatus()); - System.exit(2); - } - } - - public static void main(String[] args) throws Exception{ - Options options = DarwinPipeline.getOptions(args); - CommandLineParser parser = new DefaultParser(); - CommandLine commandLine = parser.parse(options, args); - if (commandLine.hasOption("h")|| - !commandLine.hasOption("directory")|| - !commandLine.hasOption("study") || - (!commandLine.hasOption("current_demographics_rec_count") && !commandLine.hasOption("caisis_mode"))){ - help(options, 0); - } - if (commandLine.hasOption("c")) { - launchCaisisJob(args, commandLine.getOptionValue("d"), commandLine.getOptionValue("s")); - } else { - launchJob(args, commandLine.getOptionValue("d"), commandLine.getOptionValue("s"), commandLine.getOptionValue("r")); - } - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/BatchConfiguration.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/BatchConfiguration.java deleted file mode 100644 index 3edf1d962..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/BatchConfiguration.java +++ /dev/null @@ -1,565 +0,0 @@ -/* - * Copyright (c) 2016-2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline; - -import java.net.MalformedURLException; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics.MskimpactAgeWriter; -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspineclinical.*; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline.*; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics.*; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactgeniepatient.*; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpact_medicaltherapy.*; -import org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chantclinical.*; -import org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chanttimeline.*; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; - -import java.util.*; -import javax.sql.DataSource; -import org.springframework.batch.core.*; -import org.springframework.batch.item.*; -import org.springframework.batch.core.configuration.annotation.*; -import org.springframework.context.annotation.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.support.SimpleJobLauncher; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; -import org.springframework.batch.core.step.tasklet.Tasklet; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.batch.item.support.*; -import org.springframework.batch.support.transaction.ResourcelessTransactionManager; -import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; -import org.springframework.core.io.Resource; -import org.springframework.jdbc.datasource.DriverManagerDataSource; -import org.springframework.jdbc.datasource.init.DataSourceInitializer; -import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; -import org.springframework.transaction.PlatformTransactionManager; - -/** - * - * @author jake - */ -@Configuration -@EnableBatchProcessing -@ComponentScan(basePackages="org.mskcc.cmo.ks.redcap") -public class BatchConfiguration { - - public static final String MSKIMPACT_JOB = "mskimpactJob"; - public static final String MSK_JOB = "mskJob"; - public static final String SKCM_MSKCC_2015_CHANT_JOB = "skcm_mskcc_2015_chantJob"; - public static final String MSK_CAISIS_JOB = "mskCaisisJob"; - - @Value("${darwin.chunk_size}") - private Integer chunkSize; - - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; - - @Bean - public static PropertySourcesPlaceholderConfigurer propertyConfigInDev() { - return new PropertySourcesPlaceholderConfigurer(); - } - - @Bean - public Job mskCaisisJob() { - return jobBuilderFactory.get(MSK_CAISIS_JOB) - .start(mskimpactTimelineBrainSpineStep()) - .next(mskimpactClinicalBrainSpineStep()) - .next(mskimpactEmailStep()) - .build(); - } - - @Bean - public Job mskimpactJob() { - return jobBuilderFactory.get(MSKIMPACT_JOB) - .start(mskimpactPatientDemographicsStep()) - .next(mskimpactTimelineBrainSpineStep()) - .next(mskimpactClinicalBrainSpineStep()) - //.next(mskimpactMedicalTherapyStep()) - .next(mskimpactGeniePatientClinicalStep()) - .next(mskimpactEmailStep()) - .build(); - } - - @Bean - public Job mskJob() { - return jobBuilderFactory.get(MSK_JOB) - .start(mskimpactPatientDemographicsStep()) - .build(); - } - - @Bean - public Job skcm_mskcc_2015_chantJob() { - return jobBuilderFactory.get(SKCM_MSKCC_2015_CHANT_JOB) - .start(skcm_mskcc_2015_chantClinicalStep()) - .next(skcm_mskcc_2015_chantTimelineStep()) - .build(); - } - - @Bean - public Step mskimpactPatientDemographicsStep() { - return stepBuilderFactory.get("mskimpactPatientDemographicsStep") - . chunk(chunkSize) - .reader(mskimpactPatientDemographicsReader()) - .processor(mskimpactPatientDemographicsProcessor()) - .writer(mskimpactCompositeDemographicsWriter()) - .build(); - } - - @Bean - public Step mskimpactTimelineBrainSpineStep() { - return stepBuilderFactory.get("mskimpactTimelineBrainSpineStep") - . chunk(chunkSize) - .reader(mskimpactTimelineBrainSpineReader()) - .processor(mskimpactTimelineBrainSpineProcessor()) - .writer(mskimpactTimelineBrainSpineWriter()) - .build(); - } - - @Bean - public Step mskimpactClinicalBrainSpineStep() { - return stepBuilderFactory.get("mskimpactClinicalBrainSpineStep") - . chunk(chunkSize) - .reader(readerDarwinClinicalBrainSpine()) - .processor(processorDarwinClinicalBrainSpine()) - .writer(writerDarwinClinicalBrainSpine()) - .build(); - } - - @Bean - public Step mskimpactMedicalTherapyStep() { - return stepBuilderFactory.get("mskimpactMedicalTherapyStep") - ., MskimpactMedicalTherapy> chunk(chunkSize) - .reader(mskimpactMedicalTherapyReader()) - .processor(mskimpactMedicalTherapyProcessor()) - .writer(mskimpactMedicalTherapyCompositeWriter()) - .build(); - } - - @Bean - public Step skcm_mskcc_2015_chantClinicalStep() { - return stepBuilderFactory.get("skcm_mskcc_2015_chantClinicalStep") - . chunk(chunkSize) - .reader(skcm_mskcc_2015_chantClinicalReader()) - .processor(skcm_mskcc_2015_chantClinicalCompositeProcessor()) - .writer(skcm_mskcc_2015_chantClinicalCompositeWriter()) - .build(); - } - - @Bean - public Step skcm_mskcc_2015_chantTimelineStep() { - return stepBuilderFactory.get("skcm_mskcc_2015_chantTimelineStep") - . chunk(chunkSize) - .reader(skcm_mskcc_2015_chantTimelineReader()) - .processor(skcm_mskcc_2015_chantTimelineProcessor()) - .writer(skcm_mskcc_2015_chantTimelineWriter()) - .build(); - } - - @Bean - public Step mskimpactGeniePatientClinicalStep() { - return stepBuilderFactory.get("mskimpactGeniePatientClinicalStep") - . chunk(chunkSize) - .reader(mskimpactGeniePatientReader()) - .processor(mskimpactGeniePatientProcessor()) - .writer(mskimpactGeniePatientWriter()) - .build(); - } - - @Bean - public Step mskimpactEmailStep() { - return stepBuilderFactory.get("mskimpactEmailStep") - .tasklet(darwinFetchEmailTasklet()) - .build(); - } - - @Bean - @StepScope - public Tasklet darwinFetchEmailTasklet() { - return new DarwinEmailTasklet(); - } - - @Bean - @StepScope - public DarwinUtils darwinUtils() { - return new DarwinUtils(); - } - - @Bean - @StepScope - public ItemStreamReader mskimpactPatientDemographicsReader() { - return new MskimpactPatientDemographicsReader(); - } - - @Bean - public MskimpactPatientDemographicsProcessor mskimpactPatientDemographicsProcessor() { - return new MskimpactPatientDemographicsProcessor(); - } - - @Bean - @StepScope - public ItemStreamWriter mskimpactPatientDemographicsWriter() { - return new MskimpactPatientDemographicsWriter(); - } - - @Bean - @StepScope - public ItemStreamWriter mskimpactAgeWriter() { - return new MskimpactAgeWriter(); - } - - @Bean - @StepScope - public ItemStreamWriter mskimpactVitalStatusWriter() { - return new MskimpactVitalStatusWriter(); - } - - @Bean - @StepScope - public CompositeItemWriter mskimpactCompositeDemographicsWriter() { - CompositeItemWriter writer = new CompositeItemWriter(); - List delegates = new ArrayList<>(); - delegates.add(mskimpactPatientDemographicsWriter()); - delegates.add(mskimpactAgeWriter()); - delegates.add(mskimpactVitalStatusWriter()); - writer.setDelegates(delegates); - return writer; - } - - @Bean - @StepScope - public ItemStreamReader mskimpactTimelineBrainSpineReader() { - return new MskimpactTimelineBrainSpineReader(); - } - - @Bean - @StepScope - public ItemStreamReader> mskimpactMedicalTherapyReader() { - return new MskimpactMedicalTherapyReader(); - } - - @Bean - public MskimpactMedicalTherapyProcessor mskimpactMedicalTherapyProcessor() { - return new MskimpactMedicalTherapyProcessor(); - } - - @Bean - @StepScope - public MskimpactMedicalTherapyClinicalWriter mskimpactMedicalTherapyClinicalWriter() { - return new MskimpactMedicalTherapyClinicalWriter(); - } - - @Bean - @StepScope - public MskimpactMedicalTherapyTimelineWriter mskimpactMedicalTherapyTimelineWriter() { - return new MskimpactMedicalTherapyTimelineWriter(); - } - - @Bean - @StepScope - public CompositeItemWriter mskimpactMedicalTherapyCompositeWriter() { - List writerDelegates = new ArrayList<>(); - writerDelegates.add(mskimpactMedicalTherapyClinicalWriter()); - writerDelegates.add(mskimpactMedicalTherapyTimelineWriter()); - CompositeItemWriter writer = new CompositeItemWriter<>(); - writer.setDelegates(writerDelegates); - return writer; - } - - - @Bean - @StepScope - public ItemStreamReader mskimpactGeniePatientReader() { - return new MskimpactGeniePatientReader(); - } - - @Bean - public MskimpactGeniePatientProcessor mskimpactGeniePatientProcessor() { - return new MskimpactGeniePatientProcessor(); - } - - @Bean - @StepScope - public ItemStreamWriter mskimpactGeniePatientWriter() { - return new MskimpactGeniePatientWriter(); - } - - // Beans for CompositeItemProcessor mskimpactTimelieBrainSpineProcessor delegation - // This was necessary because if a delegate processor instance is not created through a bean instance, it's members do not get Autowired - // - @Bean - public MskimpactTimelineBrainSpineModelToCompositeProcessor mskimpactTimelineBrainSpineModelToCompositeProcessorTypeStatus() { - return new MskimpactTimelineBrainSpineModelToCompositeProcessor(BrainSpineTimelineType.STATUS); - } - - @Bean - public MskimpactTimelineBrainSpineCompositeToCompositeProcessor mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeTreatment() { - return new MskimpactTimelineBrainSpineCompositeToCompositeProcessor(BrainSpineTimelineType.TREATMENT); - } - - @Bean - public MskimpactTimelineBrainSpineCompositeToCompositeProcessor mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeSurgery() { - return new MskimpactTimelineBrainSpineCompositeToCompositeProcessor(BrainSpineTimelineType.SURGERY); - } - - @Bean - public MskimpactTimelineBrainSpineCompositeToCompositeProcessor mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeSpecimen() { - return new MskimpactTimelineBrainSpineCompositeToCompositeProcessor(BrainSpineTimelineType.SPECIMEN); - } - - @Bean - public MskimpactTimelineBrainSpineCompositeToCompositeProcessor mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeImaging() { - return new MskimpactTimelineBrainSpineCompositeToCompositeProcessor(BrainSpineTimelineType.IMAGING); - } - - @Bean - @StepScope - public CompositeItemProcessor mskimpactTimelineBrainSpineProcessor() { - List processorDelegates = new ArrayList<>(); - processorDelegates.add(mskimpactTimelineBrainSpineModelToCompositeProcessorTypeStatus()); - processorDelegates.add(mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeTreatment()); - processorDelegates.add(mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeSurgery()); - processorDelegates.add(mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeSpecimen()); - processorDelegates.add(mskimpactTimelineBrainSpineCompositeToCompositeProcessorTypeImaging()); - CompositeItemProcessor processor = new CompositeItemProcessor<>(); - processor.setDelegates(processorDelegates); - return processor; - } - - @Bean - @StepScope - public ItemStreamWriter statusWriter() { - return new MskimpactTimelineBrainSpineWriter(BrainSpineTimelineType.STATUS); - } - - @Bean - @StepScope - public ItemStreamWriter specimenWriter() { - return new MskimpactTimelineBrainSpineWriter(BrainSpineTimelineType.SPECIMEN); - } - - @Bean - @StepScope - public ItemStreamWriter surgeryWriter() { - return new MskimpactTimelineBrainSpineWriter(BrainSpineTimelineType.SURGERY); - } - - @Bean - @StepScope - public ItemStreamWriter treatmentWriter() { - return new MskimpactTimelineBrainSpineWriter(BrainSpineTimelineType.TREATMENT); - } - - @Bean - @StepScope - public ItemStreamWriter imagingWriter() { - return new MskimpactTimelineBrainSpineWriter(BrainSpineTimelineType.IMAGING); - } - - @Bean - @StepScope - public CompositeItemWriter mskimpactTimelineBrainSpineWriter() { - List writerDelegates = new ArrayList<>(); - writerDelegates.add(statusWriter()); - writerDelegates.add(treatmentWriter()); - writerDelegates.add(surgeryWriter()); - writerDelegates.add(specimenWriter()); - writerDelegates.add(imagingWriter()); - CompositeItemWriter writer = new CompositeItemWriter<>(); - writer.setDelegates(writerDelegates); - return writer; - } - - @Bean - @StepScope - public ItemStreamReader readerDarwinClinicalBrainSpine() { - return new MskimpactBrainSpineClinicalReader(); - } - - @Bean - public MskimpactBrainSpineClinicalProcessor processorDarwinClinicalBrainSpine() { - return new MskimpactBrainSpineClinicalProcessor(); - } - - @Bean - @StepScope - public MskimpactBrainSpineClinicalWriter writerDarwinClinicalBrainSpine() { - return new MskimpactBrainSpineClinicalWriter(); - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantClinicalReader skcm_mskcc_2015_chantClinicalReader() { - return new Skcm_mskcc_2015_chantClinicalReader(); - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantClinicalSampleProcessor skcm_mskcc_2015_chantClinicalSampleProcessor() { - return new Skcm_mskcc_2015_chantClinicalSampleProcessor(); - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantClinicalPatientProcessor skcm_mskcc_2015_chantClinicalPatientProcessor() { - return new Skcm_mskcc_2015_chantClinicalPatientProcessor(); - } - - @Bean - @StepScope - public CompositeItemProcessor skcm_mskcc_2015_chantClinicalCompositeProcessor() { - List processorDelegates = new ArrayList<>(); - processorDelegates.add(skcm_mskcc_2015_chantClinicalSampleProcessor()); - processorDelegates.add(skcm_mskcc_2015_chantClinicalPatientProcessor()); - CompositeItemProcessor processor = new CompositeItemProcessor<>(); - processor.setDelegates(processorDelegates); - return processor; - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantClinicalSampleWriter skcm_mskcc_2015_chantClinicalSampleWriter() { - return new Skcm_mskcc_2015_chantClinicalSampleWriter(); - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantClinicalPatientWriter skcm_mskcc_2015_chantClinicalPatientWriter() { - return new Skcm_mskcc_2015_chantClinicalPatientWriter(); - } - - @Bean - @StepScope - public CompositeItemWriter skcm_mskcc_2015_chantClinicalCompositeWriter() { - List writerDelegates = new ArrayList<>(); - writerDelegates.add(skcm_mskcc_2015_chantClinicalSampleWriter()); - writerDelegates.add(skcm_mskcc_2015_chantClinicalPatientWriter()); - CompositeItemWriter writer = new CompositeItemWriter<>(); - writer.setDelegates(writerDelegates); - return writer; - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantTimelineReader skcm_mskcc_2015_chantTimelineReader() { - return new Skcm_mskcc_2015_chantTimelineReader(); - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantTimelineProcessor skcm_mskcc_2015_chantTimelineProcessor() { - return new Skcm_mskcc_2015_chantTimelineProcessor(); - } - - @Bean - @StepScope - public Skcm_mskcc_2015_chantTimelineWriter skcm_mskcc_2015_chantTimelineWriter() { - return new Skcm_mskcc_2015_chantTimelineWriter(); - } - - // general spring batch configuration - @Value("org/springframework/batch/core/schema-drop-sqlite.sql") - private Resource dropRepositoryTables; - - @Value("org/springframework/batch/core/schema-sqlite.sql") - private Resource dataRepositorySchema; - - /** - * Spring Batch datasource. - * @return DataSource - */ - @Bean - public DataSource dataSource() { - DriverManagerDataSource dataSource = new DriverManagerDataSource(); - dataSource.setDriverClassName("org.sqlite.JDBC"); - dataSource.setUrl("jdbc:sqlite:repository.sqlite"); - return dataSource; - } - - /** - * Spring Batch datasource initializer. - * @param dataSource - * @return DataSourceInitializer - * @throws MalformedURLException - */ - @Bean - public DataSourceInitializer dataSourceInitializer(DataSource dataSource) throws MalformedURLException { - ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator(); - databasePopulator.addScript(dropRepositoryTables); - databasePopulator.addScript(dataRepositorySchema); - databasePopulator.setIgnoreFailedDrops(true); - - DataSourceInitializer initializer = new DataSourceInitializer(); - initializer.setDataSource(dataSource); - initializer.setDatabasePopulator(databasePopulator); - return initializer; - } - - /** - * Spring Batch job repository. - * @return JobRepository - * @throws Exception - */ - private JobRepository getJobRepository() throws Exception { - JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean(); - factory.setDataSource(dataSource()); - factory.setTransactionManager(getTransactionManager()); - factory.afterPropertiesSet(); - return (JobRepository) factory.getObject(); - } - - /** - * Spring Batch transaction manager. - * @return PlatformTransactionManager - */ - private PlatformTransactionManager getTransactionManager() { - return new ResourcelessTransactionManager(); - } - - /** - * Spring Batch job launcher. - * @return JobLauncher - * @throws Exception - */ - public JobLauncher getJobLauncher() throws Exception { - SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); - jobLauncher.setJobRepository(getJobRepository()); - jobLauncher.afterPropertiesSet(); - return jobLauncher; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinConfiguration.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinConfiguration.java deleted file mode 100644 index 7b1f5bbc6..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinConfiguration.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline; - -import com.ibm.db2.jcc.DB2SimpleDataSource; -import com.querydsl.sql.SQLQueryFactory; -import com.querydsl.sql.DB2Templates; -import java.sql.SQLException; -import java.util.*; -import java.util.regex.Pattern; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; -import org.springframework.beans.factory.annotation.Value; -/** - * - * @author jake - */ -@Configuration -@PropertySource("classpath:application.properties") -public class DarwinConfiguration { - - @Value("${darwin.username}") - private String username; - - @Value("${darwin.password}") - private String password; - - @Value("${darwin.server}") - private String server; - - @Value("${darwin.port}") - private Integer port; - - @Value("${darwin.database}") - private String database; - - @Value("${darwin.schema}") - private String schema; - - private final Pattern MSKIMPACT_REGEX = Pattern.compile("P-\\d*-T\\d*-IM\\d*"); - private final Pattern MSKIMPACT_HEME_REGEX = Pattern.compile("P-\\d*-T\\d*-IH\\d*"); - private final Pattern MSKARCHER_REGEX = Pattern.compile("P-\\d*-T\\d*-A[H|S]\\d*"); - private final Pattern MSKRAINDANCE_REGEX = Pattern.compile("P-\\d*-T\\d*-T[B|S]\\d*"); - - @Bean - public SQLQueryFactory darwinQueryFactory() throws SQLException{ - DB2Templates templates = new DB2Templates(); - com.querydsl.sql.Configuration config = new com.querydsl.sql.Configuration(templates); - return new SQLQueryFactory(config, darwinDataSource()); - } - - public DB2SimpleDataSource darwinDataSource(){ - DB2SimpleDataSource dataSource = new DB2SimpleDataSource(); - dataSource.setPortNumber(port); - dataSource.setUser(username); - dataSource.setPassword(password); - dataSource.setDatabaseName(database); - dataSource.setCurrentSchema(schema); - dataSource.setServerName(server); - dataSource.setDriverType(4); - return dataSource; - } - - @Bean(name="studyIdRegexMap") - public Map studyIdRegexMap() { - Map studyIdRegexMap = new HashMap<>(); - studyIdRegexMap.put("mskimpact", MSKIMPACT_REGEX); - studyIdRegexMap.put("mskimpact_heme", MSKIMPACT_HEME_REGEX); - studyIdRegexMap.put("mskarcher", MSKARCHER_REGEX); - studyIdRegexMap.put("mskraindance", MSKRAINDANCE_REGEX); - return studyIdRegexMap; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinEmailTasklet.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinEmailTasklet.java deleted file mode 100644 index c4f65c3ad..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/DarwinEmailTasklet.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline; - -import org.cbioportal.cmo.pipelines.common.util.EmailUtil; - -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinSampleListUtil; - -import org.apache.log4j.Logger; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.batch.core.StepContribution; -import org.springframework.batch.core.scope.context.ChunkContext; -import org.springframework.batch.core.step.tasklet.Tasklet; -import org.springframework.batch.repeat.RepeatStatus; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.stereotype.Service; - -import java.util.*; - -/** - * - * @author Manda Wilson - */ -@Service -@ComponentScan(basePackages = {"org.mskcc.cmo.ks.darwin.pipeline", "org.cbioportal.cmo.pipelines.common"}) -public class DarwinEmailTasklet implements Tasklet { - - @Autowired - private EmailUtil emailUtil; - - @Autowired - private DarwinSampleListUtil darwinSampleListUtil; - - Logger log = Logger.getLogger(DarwinEmailTasklet.class); - - @Override - public RepeatStatus execute(StepContribution stepContribution, ChunkContext chunkContext) throws Exception { - StringBuilder body = new StringBuilder(); - String subject = "Darwin pipeline errors"; - Set filteredMskimpactBrainSpineTimelineSet = darwinSampleListUtil.getFilteredMskimpactBrainSpineTimelineSet(); - if (filteredMskimpactBrainSpineTimelineSet.size() > 0) { - body.append(constructFilteredMskimpactBrainSpineTimelineText(filteredMskimpactBrainSpineTimelineSet)); - } - if (!body.toString().isEmpty()) { - emailUtil.sendEmailToDefaultRecipient(subject, body.toString()); - } - return RepeatStatus.FINISHED; - } - - private String constructFilteredMskimpactBrainSpineTimelineText(Set filteredMskimpactBrainSpineTimelineSet) { - log.warn(filteredMskimpactBrainSpineTimelineSet.size() + " records from Darwin were filtered"); - StringBuilder text = new StringBuilder(); - text.append("\nFiltered "); - text.append(filteredMskimpactBrainSpineTimelineSet.size()); - text.append(" MSK IMPACT Caisis GBM timeline records:\n"); - for (MskimpactBrainSpineTimeline mskimpactBrainSpineTimeline : filteredMskimpactBrainSpineTimelineSet) { - text.append("\n\tPATIENT_ID: "); - text.append(mskimpactBrainSpineTimeline.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN()); - text.append(", START_DATE: "); - text.append(mskimpactBrainSpineTimeline.getSTART_DATE()); - text.append(", STOP_DATE: "); - text.append(mskimpactBrainSpineTimeline.getSTOP_DATE()); - text.append(", EVENT_TYPE: "); - text.append(mskimpactBrainSpineTimeline.getEVENT_TYPE()); - text.append(", TREATMENT_TYPE: "); - text.append(mskimpactBrainSpineTimeline.getTREATMENT_TYPE()); - } - return text.toString(); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineClinical.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineClinical.java deleted file mode 100644 index 30182f1cc..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineClinical.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author jake - */ -public class MskimpactBrainSpineClinical { - private String DMT_PATIENT_ID_BRAINSPINECLIN; - private String DMP_PATIENT_ID_BRAINSPINECLIN; - private String DMP_SAMPLE_ID_BRAINSPINECLIN; - private String AGE; - private String SEX; - private String OS_STATUS; - private String OS_MONTHS; - private String DFS_STATUS; - private String DFS_MONTHS; - private String HISTOLOGY; - private String WHO_GRADE; - private String MGMT_STATUS; - - public MskimpactBrainSpineClinical(){} - - public MskimpactBrainSpineClinical( - String DMP_PATIENT_ID_BRAINSPINECLIN, - String DMP_SAMPLE_ID_BRAINSPINECLIN, - String AGE, - String SEX, - String OS_STATUS, - String OS_MONTHS, - String DFS_STATUS, - String DFS_MONTHS, - String HISTOLOGY, - String WHO_GRADE, - String MGMT_STATUS){ - - this.DMP_PATIENT_ID_BRAINSPINECLIN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_BRAINSPINECLIN); - this.DMP_SAMPLE_ID_BRAINSPINECLIN = ClinicalValueUtil.defaultWithNA(DMP_SAMPLE_ID_BRAINSPINECLIN); - this.AGE = ClinicalValueUtil.defaultWithNA(AGE); - this.SEX = ClinicalValueUtil.defaultWithNA(SEX); - this.OS_STATUS = ClinicalValueUtil.defaultWithNA(OS_STATUS); - this.OS_MONTHS = ClinicalValueUtil.defaultWithNA(OS_MONTHS); - this.DFS_STATUS = ClinicalValueUtil.defaultWithNA(DFS_STATUS); - this.DFS_MONTHS = ClinicalValueUtil.defaultWithNA(DFS_MONTHS); - this.HISTOLOGY = ClinicalValueUtil.defaultWithNA(HISTOLOGY); - this.WHO_GRADE = ClinicalValueUtil.defaultWithNA(WHO_GRADE); - this.MGMT_STATUS = ClinicalValueUtil.defaultWithNA(MGMT_STATUS); - } - - public String getDMP_PATIENT_ID_BRAINSPINECLIN() { - return DMP_PATIENT_ID_BRAINSPINECLIN; - } - - public void setDMP_PATIENT_ID_BRAINSPINECLIN(String DMP_PATIENT_ID_BRAINSPINECLIN) { - this.DMP_PATIENT_ID_BRAINSPINECLIN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_BRAINSPINECLIN); - } - - public String getDMP_SAMPLE_ID_BRAINSPINECLIN() { - return DMP_SAMPLE_ID_BRAINSPINECLIN; - } - - public void setDMP_SAMPLE_ID_BRAINSPINECLIN(String DMP_SAMPLE_ID_BRAINSPINECLIN) { - this.DMP_SAMPLE_ID_BRAINSPINECLIN = ClinicalValueUtil.defaultWithNA(DMP_SAMPLE_ID_BRAINSPINECLIN); - } - - public String getAGE() { - return AGE; - } - - public void setAGE(String AGE) { - this.AGE = ClinicalValueUtil.defaultWithNA(AGE); - } - - public String getSEX() { - return SEX; - } - - public void setSEX(String SEX) { - this.SEX = ClinicalValueUtil.defaultWithNA(SEX); - } - - public String getOS_STATUS() { - return OS_STATUS; - } - - public void setOS_STATUS(String OS_STATUS) { - this.OS_STATUS = ClinicalValueUtil.defaultWithNA(OS_STATUS); - } - - public String getOS_MONTHS() { - return OS_MONTHS; - } - - public void setOS_MONTHS(String OS_MONTHS) { - this.OS_MONTHS = ClinicalValueUtil.defaultWithNA(OS_MONTHS); - } - - public String getDFS_STATUS() { - return DFS_STATUS; - } - - public void setDFS_STATUS(String DFS_STATUS) { - this.DFS_STATUS = ClinicalValueUtil.defaultWithNA(DFS_STATUS); - } - - public String getDFS_MONTHS() { - return DFS_MONTHS; - } - - public void setDFS_MONTHS(String DFS_MONTHS) { - this.DFS_MONTHS = ClinicalValueUtil.defaultWithNA(DFS_MONTHS); - } - - public String getHISTOLOGY() { - return HISTOLOGY; - } - - public void setHISTOLOGY(String HISTOLOGY) { - this.HISTOLOGY = ClinicalValueUtil.defaultWithNA(HISTOLOGY); - } - - public String getWHO_GRADE() { - return WHO_GRADE; - } - - public void setWHO_GRADE(String WHO_GRADE) { - this.WHO_GRADE = ClinicalValueUtil.defaultWithNA(WHO_GRADE); - } - - public String getMGMT_STATUS() { - return MGMT_STATUS; - } - - public void setMGMT_STATUS(String MGMT_STATUS) { - this.MGMT_STATUS = ClinicalValueUtil.defaultWithNA(MGMT_STATUS); - } - - @Override - public String toString(){ - return ToStringBuilder.reflectionToString(this); - } - - public List getFieldNames(){ - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_SAMPLE_ID_BRAINSPINECLIN"); - fieldNames.add("AGE"); - fieldNames.add("SEX"); - fieldNames.add("OS_STATUS"); - fieldNames.add("OS_MONTHS"); - fieldNames.add("DFS_STATUS"); - fieldNames.add("DFS_MONTHS"); - fieldNames.add("HISTOLOGY"); - fieldNames.add("WHO_GRADE"); - fieldNames.add("MGMT_STATUS"); - fieldNames.add("DMP_PATIENT_ID_BRAINSPINECLIN"); - - return fieldNames; - } - - public List getHeaders(){ - List fieldNames = new ArrayList<>(); - fieldNames.add("SAMPLE_ID"); - fieldNames.add("AGE"); - fieldNames.add("SEX_DMT"); - fieldNames.add("OS_STATUS_DMT"); - fieldNames.add("OS_MONTHS_DMT"); - fieldNames.add("DFS_STATUS"); - fieldNames.add("DFS_MONTHS"); - fieldNames.add("HISTOLOGICAL_DIAGNOSIS"); - fieldNames.add("WHO_GRADE"); - fieldNames.add("MGMT_STATUS"); - fieldNames.add("PATIENT_ID"); - - return fieldNames; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineCompositeTimeline.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineCompositeTimeline.java deleted file mode 100644 index 983277129..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineCompositeTimeline.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; - -/** - * - * @author jake - */ - -public class MskimpactBrainSpineCompositeTimeline { - - private String statusResult; - private String specimenResult; - private String treatmentResult; - private String imagingResult; - private String surgeryResult; - private MskimpactBrainSpineTimeline record; - - public static final String NO_RESULT = "NO RESULT"; - - public MskimpactBrainSpineCompositeTimeline(MskimpactBrainSpineTimeline record){ - this.statusResult = NO_RESULT; - this.specimenResult = NO_RESULT; - this.treatmentResult = NO_RESULT; - this.imagingResult = NO_RESULT; - this.surgeryResult = NO_RESULT; - this.record = record; - } - - public String getSurgeryResult(){ - return surgeryResult; - } - - public void setSurgeryResult(String s){ - this.surgeryResult = s; - } - - public String getImagingResult(){ - return imagingResult; - } - - public void setImagingResult(String s){ - this.imagingResult = s; - } - - public String getStatusResult(){ - return statusResult; - } - - public void setStatusResult(String s){ - this.statusResult = s; - } - - public String getTreatmentResult(){ - return treatmentResult; - } - - public void setTreatmentResult(String s){ - this.treatmentResult = s; - } - - public String getSpecimenResult(){ - return specimenResult; - } - - public void setSpecimenResult(String s){ - this.specimenResult = s; - } - - public MskimpactBrainSpineTimeline getRecord(){ - return this.record; - } - - public void setRecord(MskimpactBrainSpineTimeline record){ - this.record = record; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineTimeline.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineTimeline.java deleted file mode 100644 index 468407b4b..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactBrainSpineTimeline.java +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author jake - */ -public class MskimpactBrainSpineTimeline { - private String DMT_PATIENT_ID_BRAINSPINETMLN; - private String DMP_PATIENT_ID_MIN_BRAINSPINETMLN; - private String DMP_PATIENT_ID_MAX_BRAINSPINETMLN; - private String DMP_PATIENT_ID_COUNT_BRAINSPINETMLN; - private String DMP_PATIENT_ID_ALL_BRAINSPINETMLN; - private String START_DATE; - private String STOP_DATE; - private String EVENT_TYPE; - private String TREATMENT_TYPE; - private String SUBTYPE; - private String AGENT; - private String SPECIMEN_REFERENCE_NUMBER; - private String SPECIMEN_SITE; - private String SPECIMEN_TYPE; - private String STATUS; - private String KARNOFSKY_PERFORMANCE_SCORE; - private String SURGERY_DETAILS; - private String EVENT_TYPE_DETAILED; - private String HISTOLOGY; - private String WHO_GRADE; - private String MGMT_STATUS; - private String SOURCE_PATHOLOGY; - private String NOTE; - private String DIAGNOSTIC_TYPE; - private String DIAGNOSTIC_TYPE_DETAILED; - private String SOURCE; - private Map additionalProperties = new HashMap<>(); - - public MskimpactBrainSpineTimeline() {} - - public MskimpactBrainSpineTimeline( - String DMT_PATIENT_ID_BRAINSPINETMLN, - String DMP_PATIENT_ID_MIN_BRAINSPINETMLN, - String DMP_PATIENT_ID_MAX_BRAINSPINETMLN, - String DMP_PATIENT_ID_COUNT_BRAINSPINETMLN, - String DMP_PATIENT_ID_ALL_BRAINSPINETMLN, - String START_DATE, - String STOP_DATE, - String EVENT_TYPE, - String TREATMENT_TYPE, - String SUBTYPE, - String AGENT, - String SPECIMEN_REFERENCE_NUMBER, - String SPECIMEN_SITE, - String SPECIMEN_TYPE, - String STATUS, - String KARNOFSKY_PERFORMANCE_SCORE, - String SURGERY_DETAILS, - String EVENT_TYPE_DETAILED, - String HISTOLOGY, - String WHO_GRADE, - String MGMT_STATUS, - String SOURCE_PATHOLOGY, - String NOTE, - String DIAGNOSTIC_TYPE, - String DIAGNOSTIC_TYPE_DETAILED, - String SOURCE) { - this.DMT_PATIENT_ID_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMT_PATIENT_ID_BRAINSPINETMLN); - this.DMP_PATIENT_ID_MIN_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_MIN_BRAINSPINETMLN); - this.DMP_PATIENT_ID_MAX_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_MAX_BRAINSPINETMLN); - this.DMP_PATIENT_ID_COUNT_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_COUNT_BRAINSPINETMLN); - this.DMP_PATIENT_ID_ALL_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_ALL_BRAINSPINETMLN); - this.START_DATE = ClinicalValueUtil.defaultWithNA(START_DATE); - this.STOP_DATE = ClinicalValueUtil.defaultWithNA(STOP_DATE); - this.EVENT_TYPE = ClinicalValueUtil.defaultWithNA(EVENT_TYPE); - this.TREATMENT_TYPE = ClinicalValueUtil.defaultWithNA(TREATMENT_TYPE); - this.SUBTYPE = ClinicalValueUtil.defaultWithNA(SUBTYPE); - this.AGENT = ClinicalValueUtil.defaultWithNA(AGENT); - this.SPECIMEN_REFERENCE_NUMBER = ClinicalValueUtil.defaultWithNA(SPECIMEN_REFERENCE_NUMBER); - this.SPECIMEN_SITE = ClinicalValueUtil.defaultWithNA(SPECIMEN_SITE); - this.SPECIMEN_TYPE = ClinicalValueUtil.defaultWithNA(SPECIMEN_TYPE); - this.STATUS = ClinicalValueUtil.defaultWithNA(STATUS); - this.KARNOFSKY_PERFORMANCE_SCORE = ClinicalValueUtil.defaultWithNA(KARNOFSKY_PERFORMANCE_SCORE); - this.SURGERY_DETAILS = ClinicalValueUtil.defaultWithNA(SURGERY_DETAILS); - this.EVENT_TYPE_DETAILED = ClinicalValueUtil.defaultWithNA(EVENT_TYPE_DETAILED); - this.HISTOLOGY = ClinicalValueUtil.defaultWithNA(HISTOLOGY); - this.WHO_GRADE = ClinicalValueUtil.defaultWithNA(WHO_GRADE); - this.MGMT_STATUS = ClinicalValueUtil.defaultWithNA(MGMT_STATUS); - this.SOURCE_PATHOLOGY = ClinicalValueUtil.defaultWithNA(SOURCE_PATHOLOGY); - this.NOTE = ClinicalValueUtil.defaultWithNA(NOTE); - this.DIAGNOSTIC_TYPE = ClinicalValueUtil.defaultWithNA(DIAGNOSTIC_TYPE); - this.DIAGNOSTIC_TYPE_DETAILED = ClinicalValueUtil.defaultWithNA(DIAGNOSTIC_TYPE_DETAILED); - this.SOURCE = ClinicalValueUtil.defaultWithNA(SOURCE); - } - - public String getDMT_PATIENT_ID_BRAINSPINETMLN() { - return DMT_PATIENT_ID_BRAINSPINETMLN; - } - - public void setDMT_PATIENT_ID_BRAINSPINETMLN(String DMT_PATIENT_ID_BRAINSPINETMLN) { - this.DMT_PATIENT_ID_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMT_PATIENT_ID_BRAINSPINETMLN); - } - - public String getDMP_PATIENT_ID_MIN_BRAINSPINETMLN() { - return DMP_PATIENT_ID_MIN_BRAINSPINETMLN; - } - - public void setDMP_PATIENT_ID_MIN_BRAINSPINETMLN(String DMP_PATIENT_ID_MIN_BRAINSPINETMLN) { - this.DMP_PATIENT_ID_MIN_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_MIN_BRAINSPINETMLN); - } - - public String getDMP_PATIENT_ID_MAX_BRAINSPINETMLN() { - return DMP_PATIENT_ID_MAX_BRAINSPINETMLN; - } - - public void setDMP_PATIENT_ID_MAX_BRAINSPINETMLN(String DMP_PATIENT_ID_MAX_BRAINSPINETMLN) { - this.DMP_PATIENT_ID_MAX_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_MAX_BRAINSPINETMLN); - } - - public String getDMP_PATIENT_ID_COUNT_BRAINSPINETMLN() { - return DMP_PATIENT_ID_COUNT_BRAINSPINETMLN; - } - - public void setDMP_PATIENT_ID_COUNT_BRAINSPINETMLN(String DMP_PATIENT_ID_COUNT_BRAINSPINETMLN) { - this.DMP_PATIENT_ID_COUNT_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_COUNT_BRAINSPINETMLN); - } - - public String getDMP_PATIENT_ID_ALL_BRAINSPINETMLN() { - return DMP_PATIENT_ID_ALL_BRAINSPINETMLN; - } - - public void setDMP_PATIENT_ID_ALL_BRAINSPINETMLN(String DMP_PATIENT_ID_ALL_BRAINSPINETMLN) { - this.DMP_PATIENT_ID_ALL_BRAINSPINETMLN = ClinicalValueUtil.defaultWithNA(DMP_PATIENT_ID_ALL_BRAINSPINETMLN); - } - - public String getSTART_DATE() { - return START_DATE; - } - - public void setSTART_DATE(String START_DATE) { - this.START_DATE = ClinicalValueUtil.defaultWithNA(START_DATE); - } - - public String getSTOP_DATE() { - return STOP_DATE; - } - - public void setSTOP_DATE(String STOP_DATE) { - this.STOP_DATE = ClinicalValueUtil.defaultWithNA(STOP_DATE); - } - - public String getEVENT_TYPE() { - return EVENT_TYPE; - } - - public void setEVENT_TYPE(String EVENT_TYPE) { - this.EVENT_TYPE = ClinicalValueUtil.defaultWithNA(EVENT_TYPE); - } - - public String getTREATMENT_TYPE() { - return TREATMENT_TYPE; - } - - public void setTREATMENT_TYPE(String TREATMENT_TYPE) { - this.TREATMENT_TYPE = ClinicalValueUtil.defaultWithNA(TREATMENT_TYPE); - } - - public String getSUBTYPE() { - return SUBTYPE; - } - - public void setSUBTYPE(String SUBTYPE) { - this.SUBTYPE = ClinicalValueUtil.defaultWithNA(SUBTYPE); - } - - public String getAGENT() { - return AGENT; - } - - public void setAGENT(String AGENT) { - this.AGENT = ClinicalValueUtil.defaultWithNA(AGENT); - } - - public String getSPECIMEN_REFERENCE_NUMBER() { - return SPECIMEN_REFERENCE_NUMBER; - } - - public void setSPECIMEN_REFERENCE_NUMBER(String SPECIMEN_REFERENCE_NUMBER) { - this.SPECIMEN_REFERENCE_NUMBER = ClinicalValueUtil.defaultWithNA(SPECIMEN_REFERENCE_NUMBER); - } - - public String getSPECIMEN_SITE() { - return SPECIMEN_SITE; - } - - public void setSPECIMEN_SITE(String SPECIMEN_SITE) { - this.SPECIMEN_SITE = ClinicalValueUtil.defaultWithNA(SPECIMEN_SITE); - } - - public String getSPECIMEN_TYPE() { - return SPECIMEN_TYPE; - } - - public void setSPECIMEN_TYPE(String SPECIMEN_TYPE) { - this.SPECIMEN_TYPE = ClinicalValueUtil.defaultWithNA(SPECIMEN_TYPE); - } - - public String getSTATUS() { - return STATUS; - } - - public void setSTATUS(String STATUS) { - this.STATUS = ClinicalValueUtil.defaultWithNA(STATUS); - } - - public String getKARNOFSKY_PERFORMANCE_SCORE() { - return KARNOFSKY_PERFORMANCE_SCORE; - } - - public void setKARNOFSKY_PERFORMANCE_SCORE(String KARNOFSKY_PERFORMANCE_SCORE) { - this.KARNOFSKY_PERFORMANCE_SCORE = ClinicalValueUtil.defaultWithNA(KARNOFSKY_PERFORMANCE_SCORE); - } - - public String getSURGERY_DETAILS() { - return SURGERY_DETAILS; - } - - public void setSURGERY_DETAILS(String SURGERY_DETAILS) { - this.SURGERY_DETAILS = ClinicalValueUtil.defaultWithNA(SURGERY_DETAILS); - } - - public String getEVENT_TYPE_DETAILED() { - return EVENT_TYPE_DETAILED; - } - - public void setEVENT_TYPE_DETAILED(String EVENT_TYPE_DETAILED) { - this.EVENT_TYPE_DETAILED = ClinicalValueUtil.defaultWithNA(EVENT_TYPE_DETAILED); - } - - public String getHISTOLOGY() { - return HISTOLOGY; - } - - public void setHISTOLOGY(String HISTOLOGY) { - this.HISTOLOGY = ClinicalValueUtil.defaultWithNA(HISTOLOGY); - } - - public String getWHO_GRADE() { - return WHO_GRADE; - } - - public void setWHO_GRADE(String WHO_GRADE) { - this.WHO_GRADE = ClinicalValueUtil.defaultWithNA(WHO_GRADE); - } - - public String getMGMT_STATUS() { - return MGMT_STATUS; - } - - public void setMGMT_STATUS(String MGMT_STATUS) { - this.MGMT_STATUS = ClinicalValueUtil.defaultWithNA(MGMT_STATUS); - } - - public String getSOURCE_PATHOLOGY() { - return SOURCE_PATHOLOGY; - } - - public void setSOURCE_PATHOLOGY(String SOURCE_PATHOLOGY) { - this.SOURCE_PATHOLOGY = ClinicalValueUtil.defaultWithNA(SOURCE_PATHOLOGY); - } - - public String getNOTE() { - return NOTE; - } - - public void setNOTE(String NOTE) { - this.NOTE = ClinicalValueUtil.defaultWithNA(NOTE); - } - - public String getDIAGNOSTIC_TYPE() { - return DIAGNOSTIC_TYPE; - } - - public void setDIAGNOSTIC_TYPE(String DIAGNOSTIC_TYPE) { - this.DIAGNOSTIC_TYPE = ClinicalValueUtil.defaultWithNA(DIAGNOSTIC_TYPE); - } - - public String getDIAGNOSTIC_TYPE_DETAILED() { - return DIAGNOSTIC_TYPE_DETAILED; - } - - public void setDIAGNOSTIC_TYPE_DETAILED(String DIAGNOSTIC_TYPE_DETAILED) { - this.DIAGNOSTIC_TYPE_DETAILED = ClinicalValueUtil.defaultWithNA(DIAGNOSTIC_TYPE_DETAILED); - } - - public String getSOURCE() { - return SOURCE; - } - - public void setSOURCE(String SOURCE) { - this.SOURCE = ClinicalValueUtil.defaultWithNA(SOURCE); - } - - public Map getAditionalProperties() { - return this.additionalProperties; - } - - public void setAdditionalProperty(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - - public List getStatusFields() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_PATIENT_ID_ALL_BRAINSPINETMLN"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("STATUS"); - fieldNames.add("KARNOFSKY_PERFORMANCE_SCORE"); - fieldNames.add("NOTE"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getStatusHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("STATUS"); - fieldNames.add("KARNOFSKY_PERFORMANCE_SCORE"); - fieldNames.add("NOTE"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getSpecimenFields() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_PATIENT_ID_ALL_BRAINSPINETMLN"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("SPECIMEN_REFERENCE_NUMBER"); - fieldNames.add("SPECIMEN_SITE"); - fieldNames.add("SPECIMEN_TYPE"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getSpecimenHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("SAMPLE_ID"); // specimen reference number should be sample id - fieldNames.add("SPECIMEN_SITE"); - fieldNames.add("SPECIMEN_TYPE"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getTreatmentFields() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_PATIENT_ID_ALL_BRAINSPINETMLN"); - fieldNames.add("START_DATE"); - fieldNames.add("STOP_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("TREATMENT_TYPE"); - fieldNames.add("SUBTYPE"); - fieldNames.add("AGENT"); - fieldNames.add("NOTE"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getTreatmentHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("STOP_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("TREATMENT_TYPE"); - fieldNames.add("SUBTYPE"); - fieldNames.add("AGENT"); - fieldNames.add("NOTE"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getImagingFields() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_PATIENT_ID_ALL_BRAINSPINETMLN"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("DIAGNOSTIC_TYPE"); - fieldNames.add("DIAGNOSTIC_TYPE_DETAILED"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getImagingHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("DIAGNOSTIC_TYPE"); - fieldNames.add("DIAGNOSTIC_TYPE_DETAILED"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getSurgeryFields() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_PATIENT_ID_ALL_BRAINSPINETMLN"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("MGMT_STATUS"); - fieldNames.add("SURGERY_DETAILS"); - fieldNames.add("EVENT_TYPE_DETAILED"); - fieldNames.add("HISTOLOGY"); - fieldNames.add("WHO_GRADE"); - fieldNames.add("SOURCE_PATHOLOGY"); - fieldNames.add("SOURCE"); - return fieldNames; - } - - public List getSurgeryHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("MGMT_STATUS"); - fieldNames.add("SURGERY_DETAILS"); - fieldNames.add("EVENT_TYPE_DETAILED"); - fieldNames.add("HISTOLOGY"); - fieldNames.add("WHO_GRADE"); - fieldNames.add("SOURCE_PATHOLOGY"); - fieldNames.add("SOURCE"); - return fieldNames; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactCompositeDemographics.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactCompositeDemographics.java deleted file mode 100644 index efe95ed3d..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactCompositeDemographics.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author ochoaa - */ -public class MskimpactCompositeDemographics { - private String demographicsResult; - private String ageResult; - private String vitalStatusResult; - - public MskimpactCompositeDemographics(){} - - /** - * @return the demographicsResult - */ - public String getDemographicsResult() { - return demographicsResult; - } - - /** - * @param demographicsResult the demographicsResult to set - */ - public void setDemographicsResult(String demographicsResult) { - this.demographicsResult = demographicsResult; - } - - /** - * @return the ageResult - */ - public String getAgeResult() { - return ageResult; - } - - /** - * @param ageResult the ageResult to set - */ - public void setAgeResult(String ageResult) { - this.ageResult = ageResult; - } - - /** - * @return the vitalStatusResult - */ - public String getVitalStatusResult() { - return vitalStatusResult; - } - - /** - * @param vitalStatusResult the vitalStatusResult to set - */ - public void setVitalStatusResult(String vitalStatusResult) { - this.vitalStatusResult = vitalStatusResult; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactGenieClinical.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactGenieClinical.java deleted file mode 100644 index 153424e4f..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactGenieClinical.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2016-2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; - -/** - * - * @author heinsz - */ -public class MskimpactGenieClinical { - - private String patientId; - private String sampleId; - private String ageAtSeqReport; - - public MskimpactGenieClinical() {} - - public MskimpactGenieClinical(String patientId, String sampleId, String ageAtSeqReport) { - this.patientId = patientId; - this.sampleId = sampleId; - this.ageAtSeqReport = ageAtSeqReport; - } - - public String getPatientId() { - return patientId; - } - - public void setPatientId(String patientId) { - this.patientId = patientId; - } - - public String getSampleId() { - return sampleId; - } - - public void setSampleId(String sampleId) { - this.sampleId = sampleId; - } - - public String getAgeAtSeqReport() { - return ageAtSeqReport; - } - - public void setAgeAtSeqReport(String ageAtSeqReport) { - this.ageAtSeqReport = ageAtSeqReport; - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PatientId"); - fieldNames.add("SampleId"); - fieldNames.add("AgeAtSeqReport"); - return fieldNames; - } - - public List getHeaders() { - List headerNames = new ArrayList<>(); - headerNames.add("PATIENT_ID"); - headerNames.add("SAMPLE_ID"); - headerNames.add("AGE_AT_SEQ_REPORT"); - return headerNames; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactLatestActivity.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactLatestActivity.java deleted file mode 100644 index 228df9dcb..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactLatestActivity.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class MskimpactLatestActivity { - private Integer ptIdPla; - private String dmpIdPla; - private Integer lastAppointmentYear; - private Integer lastDrVisitYear; - private Integer lastInpatientAdmissionYear; - private Integer lastInpatientDischargeYear; - private Integer lastContactYear; - private Integer lastActivityYear; - private Integer lastMskChemoYear; - private Integer lastMskRadiationTherapyYear; - private Integer lastMskSurgeryYear; - private Integer lastMskTreatmentYear; - private Integer lastKnowAliveYear; - private Integer ageAtLastAppointmentYearInDays; - private Integer ageAtLastDrVisitYearInDays; - private Integer ageAtLastInpatientAdmissionYearInDays; - private Integer ageAtLastInpatientDischargeYearInDays; - private Integer ageAtLastContactYearInDays; - private Integer lastActivityYearInDays; - private Integer ageAtLastMskChemoYearInDays; - private Integer ageAtLastMskRadiationTherapy; - private Integer ageAtLastMskSurgeryYearInDays; - private Integer ageAtLastMskTreatmentYear; - private Integer ageAtLastKnownAliveYearInDays; - - public MskimpactLatestActivity() {} - - public Integer getPT_ID_PLA() { - return ptIdPla; - } - - public void setPT_ID_PLA(Integer ptIdPla) { - this.ptIdPla = ptIdPla; - } - - public String getDMP_ID_PLA() { - return dmpIdPla; - } - - public void setDMP_ID_PLA(String dmpIdPla) { - this.dmpIdPla = dmpIdPla; - } - - public Integer getLAST_APPOINTMENT_YEAR() { - return lastAppointmentYear; - } - - public void setLAST_APPOINTMENT_YEAR(Integer lastAppointmentYear) { - this.lastAppointmentYear = lastAppointmentYear; - } - - public Integer getLAST_DR_VISIT_YEAR() { - return lastDrVisitYear; - } - - public void setLAST_DR_VISIT_YEAR(Integer lastDrVisitYear) { - this.lastDrVisitYear = lastDrVisitYear; - } - - public Integer getLAST_INPATIENT_ADMISSION_YEAR() { - return lastInpatientAdmissionYear; - } - - public void setLAST_INPATIENT_ADMISSION_YEAR(Integer lastInpatientAdmissionYear) { - this.lastInpatientAdmissionYear = lastInpatientAdmissionYear; - } - - public Integer getLAST_INPATIENT_DISCHARGE_YEAR() { - return lastInpatientDischargeYear; - } - - public void setLAST_INPATIENT_DISCHARGE_YEAR(Integer lastInpatientDischargeYear) { - this.lastInpatientDischargeYear = lastInpatientDischargeYear; - } - - public Integer getLAST_CONTACT_YEAR() { - return lastContactYear; - } - - public void setLAST_CONTACT_YEAR(Integer lastContactYear) { - this.lastContactYear = lastContactYear; - } - - public Integer getLAST_ACTIVITY_YEAR() { - return lastActivityYear; - } - - public void setLAST_ACTIVITY_YEAR(Integer lastActivityYear) { - this.lastActivityYear = lastActivityYear; - } - - public Integer getLAST_MSK_CHEMO_YEAR() { - return lastMskChemoYear; - } - - public void setLAST_MSK_CHEMO_YEAR(Integer lastMskChemoYear) { - this.lastMskChemoYear = lastMskChemoYear; - } - - public Integer getLAST_MSK_RADIATION_THERAPY_YEAR() { - return lastMskRadiationTherapyYear; - } - - public void setLAST_MSK_RADIATION_THERAPY_YEAR(Integer lastMskRadiationTherapyYear) { - this.lastMskRadiationTherapyYear = lastMskRadiationTherapyYear; - } - - public Integer getLAST_MSK_SURGERY_YEAR() { - return lastMskSurgeryYear; - } - - public void setLAST_MSK_SURGERY_YEAR(Integer lastMskSurgeryYear) { - this.lastMskSurgeryYear = lastMskSurgeryYear; - } - - public Integer getLAST_MSK_TREATMENT_YEAR() { - return lastMskTreatmentYear; - } - - public void setLAST_MSK_TREATMENT_YEAR(Integer lastMskTreatmentYear) { - this.lastMskTreatmentYear = lastMskTreatmentYear; - } - - public Integer getLAST_KNOWN_ALIVE_YEAR() { - return lastKnowAliveYear; - } - - public void setLAST_KNOWN_ALIVE_YEAR(Integer lastKnowAliveYear) { - this.lastKnowAliveYear = lastKnowAliveYear; - } - - public Integer getAGE_AT_LAST_APPOINTMENT_YEAR_IN_DAYS() { - return ageAtLastAppointmentYearInDays; - } - - public void setAGE_AT_LAST_APPOINTMENT_YEAR_IN_DAYS(Integer ageAtLastAppointmentYearInDays) { - this.ageAtLastAppointmentYearInDays = ageAtLastAppointmentYearInDays; - } - - public Integer getAGE_AT_LAST_DR_VISIT_YEAR_IN_DAYS() { - return ageAtLastDrVisitYearInDays; - } - - public void setAGE_AT_LAST_DR_VISIT_YEAR_IN_DAYS(Integer ageAtLastDrVisitYearInDays) { - this.ageAtLastDrVisitYearInDays = ageAtLastDrVisitYearInDays; - } - - public Integer getAGE_AT_LAST_INPATIENT_ADMISSION_YEAR_IN_DAYS() { - return ageAtLastInpatientAdmissionYearInDays; - } - - public void setAGE_AT_LAST_INPATIENT_ADMISSION_YEAR_IN_DAYS(Integer ageAtLastInpatientAdmissionYearInDays) { - this.ageAtLastInpatientAdmissionYearInDays = ageAtLastInpatientAdmissionYearInDays; - } - - public Integer getAGE_AT_LAST_INPATIENT_DISCHARGE_YEAR_IN_DAYS() { - return ageAtLastInpatientDischargeYearInDays; - } - - public void setAGE_AT_LAST_INPATIENT_DISCHARGE_YEAR_IN_DAYS(Integer ageAtLastInpatientDischargeYearInDays) { - this.ageAtLastInpatientDischargeYearInDays = ageAtLastInpatientDischargeYearInDays; - } - - public Integer getAGE_AT_LAST_CONTACT_YEAR_IN_DAYS() { - return ageAtLastContactYearInDays; - } - - public void setAGE_AT_LAST_CONTACT_YEAR_IN_DAYS(Integer ageAtLastContactYearInDays) { - this.ageAtLastContactYearInDays = ageAtLastContactYearInDays; - } - - public Integer getLAST_ACTIVITY_YEAR_IN_DAYS() { - return lastActivityYearInDays; - } - - public void setLAST_ACTIVITY_YEAR_IN_DAYS(Integer lastActivityYearInDays) { - this.lastDrVisitYear = lastActivityYearInDays; - } - - public Integer getAGE_AT_LAST_MSK_CHEMO_YEAR_IN_DAYS() { - return ageAtLastMskChemoYearInDays; - } - - public void setAGE_AT_LAST_MSK_CHEMO_YEAR_IN_DAYS(Integer ageAtLastMskChemoYearInDays) { - this.ageAtLastMskChemoYearInDays = ageAtLastMskChemoYearInDays; - } - - public Integer getAGE_AT_LAST_MSK_RADIATION_THERAPY() { - return ageAtLastMskRadiationTherapy; - } - - public void setAGE_AT_LAST_MSK_RADIATION_THERAPY(Integer ageAtLastMskRadiationTherapy) { - this.ageAtLastMskRadiationTherapy = ageAtLastMskRadiationTherapy; - } - - public Integer getAGE_AT_LAST_MSK_SURGERY_YEAR_IN_DAYS() { - return ageAtLastMskSurgeryYearInDays; - } - - public void setAGE_AT_LAST_MSK_SURGERY_YEAR_IN_DAYS(Integer ageAtLastMskSurgeryYearInDays) { - this.ageAtLastMskSurgeryYearInDays = ageAtLastMskSurgeryYearInDays; - } - - public Integer getAGE_AT_LAST_MSK_TREATMENT_YEAR() { - return ageAtLastMskTreatmentYear; - } - - public void setAGE_AT_LAST_MSK_TREATMENT_YEAR(Integer ageAtLastMskTreatmentYear) { - this.ageAtLastMskTreatmentYear = ageAtLastMskTreatmentYear; - } - - public Integer getAGE_AT_LAST_KNOWN_ALIVE_YEAR_IN_DAYS() { - return ageAtLastKnownAliveYearInDays; - } - - public void setAGE_AT_LAST_KNOWN_ALIVE_YEAR_IN_DAYS(Integer ageAtLastKnownAliveYearInDays) { - this.ageAtLastKnownAliveYearInDays = ageAtLastKnownAliveYearInDays; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactMedicalTherapy.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactMedicalTherapy.java deleted file mode 100644 index 9af7438c5..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactMedicalTherapy.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 2of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; - -/** - * Class to model relevant data from record within DVCBIO.PHARMACY_V. - * - * @author Benjamin Gross - */ -public class MskimpactMedicalTherapy implements Comparable -{ - private String PT_ID_PHARMACY; - private String DMP_ID_PHARMACY; - private Integer AGE_AT_DISPENSE_DATE_IN_DAYS; - private String GENERIC_DRUG_NAME; - private Float DOSAGE; - private String DOSE_UNIT; - private Float DISPENSED_QUANTITY; - private Integer START_DATE; - private Integer STOP_DATE; - private String SAMPLE_ID_PATH_DMP; - - public MskimpactMedicalTherapy() {} - - /** - * All fields are coming directly from DVCBIO.PHARMACY_V - */ - public MskimpactMedicalTherapy(String PT_ID_PHARMACY, String DMP_ID_PHARMACY, - Integer AGE_AT_DISPENSE_DATE_IN_DAYS, String GENERIC_DRUG_NAME, - Float DOSAGE, String DOSE_UNIT, Float DISPENSED_QUANTITY, String SAMPLE_ID_PATH_DMP) { - this(PT_ID_PHARMACY, DMP_ID_PHARMACY, AGE_AT_DISPENSE_DATE_IN_DAYS, - GENERIC_DRUG_NAME, DOSAGE, DOSE_UNIT, DISPENSED_QUANTITY, - AGE_AT_DISPENSE_DATE_IN_DAYS, AGE_AT_DISPENSE_DATE_IN_DAYS,SAMPLE_ID_PATH_DMP); - } - - /** - * All fields are coming from DVCBIO.PHARMACY_V with the exception of - * START_DATE, STOP_DATE which are chosen "AGE_AT_DISPENSE_DATE_IN_DAYS" of patient - * - see MksimpactMedicalTherapyProcessor for more information. - */ - public MskimpactMedicalTherapy(String PT_ID_PHARMACY, String DMP_ID_PHARMACY, - Integer AGE_AT_DISPENSE_DATE_IN_DAYS, String GENERIC_DRUG_NAME, - Float DOSAGE, String DOSE_UNIT, Float DISPENSED_QUANTITY, - Integer START_DATE, Integer STOP_DATE, String SAMPLE_ID_PATH_DMP) { - this.PT_ID_PHARMACY = PT_ID_PHARMACY.trim(); - this.DMP_ID_PHARMACY = DMP_ID_PHARMACY.trim(); - this.AGE_AT_DISPENSE_DATE_IN_DAYS = AGE_AT_DISPENSE_DATE_IN_DAYS; - this.GENERIC_DRUG_NAME = GENERIC_DRUG_NAME.trim(); - this.DOSAGE = DOSAGE; - this.DOSE_UNIT = DOSE_UNIT.trim(); - this.DISPENSED_QUANTITY = DISPENSED_QUANTITY; - this.START_DATE = START_DATE; - this.STOP_DATE = STOP_DATE; - this.SAMPLE_ID_PATH_DMP = SAMPLE_ID_PATH_DMP; - } - - public String getPT_ID_PHARMACY() { - return PT_ID_PHARMACY; - } - - public String getDMP_ID_PHARMACY() { - return DMP_ID_PHARMACY; - } - - public Integer getAGE_AT_DISPENSE_DATE_IN_DAYS() { - return AGE_AT_DISPENSE_DATE_IN_DAYS; - } - - public String getGENERIC_DRUG_NAME() { - return GENERIC_DRUG_NAME; - } - - public Float getDOSAGE() { - return DOSAGE; - } - - public String getDOSE_UNIT() { - return DOSE_UNIT; - } - - public Float getDISPENSED_QUANTITY() { - return DISPENSED_QUANTITY; - } - - public Integer getSTART_DATE() { - return START_DATE; - } - - public Integer getSTOP_DATE() { - return STOP_DATE; - } - - /** - * @return the SAMPLE_ID_PATH_DMP - */ - public String getSAMPLE_ID_PATH_DMP() { - return SAMPLE_ID_PATH_DMP; - } - - /** - * @param SAMPLE_ID_PATH_DMP the SAMPLE_ID_PATH_DMP to set - */ - public void setSAMPLE_ID_PATH_DMP(String SAMPLE_ID_PATH_DMP) { - this.SAMPLE_ID_PATH_DMP = SAMPLE_ID_PATH_DMP; - } - - public static List getHeaders(){ - return Arrays.asList("PATIENT_ID", "START_DATE", "STOP_DATE", "EVENT_TYPE", - "TREATMENT_TYPE", "SUBTYPE", "AGENT", "DOSAGE"); - } - - /** - * MskimpactMedicalTherapyProcessor will sort a collection of these records - * by AGE_AT_DISPENSE_DATE_IN_DAYS in order to create a START_DATE-STOP_DATE range. - */ - @Override - public int compareTo(MskimpactMedicalTherapy that) { - if (this.AGE_AT_DISPENSE_DATE_IN_DAYS < that.AGE_AT_DISPENSE_DATE_IN_DAYS) return -1; - if (this.AGE_AT_DISPENSE_DATE_IN_DAYS > that.AGE_AT_DISPENSE_DATE_IN_DAYS) return 1; - return 0; - } - - /** - * This is called to create a record with within the timeline file - */ - @Override - public String toString() { - return (DMP_ID_PHARMACY + "\t" + START_DATE + "\t" + STOP_DATE + "\t" + - "TREATMENT\tMedical Therapy\tImmunotherapy\t" + - GENERIC_DRUG_NAME + "\t" + Math.round(DOSAGE) + " " + DOSE_UNIT); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactNAACCRClinical.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactNAACCRClinical.java deleted file mode 100644 index df117e3aa..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactNAACCRClinical.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 2of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author heinsz - */ -public class MskimpactNAACCRClinical { - private String DMP_ID_DEMO; - private String PT_NAACCR_SEX_CODE; - private String PT_NAACCR_RACE_CODE_PRIMARY; - private String PT_NAACCR_RACE_CODE_SECONDARY; - private String PT_NAACCR_RACE_CODE_TERTIARY; - private String PT_NAACCR_ETHNICITY_CODE; - private String PT_BIRTH_YEAR; - - public MskimpactNAACCRClinical() {} - - public MskimpactNAACCRClinical(String DMP_ID_DEMO, String PT_NAACCR_SEX_CODE, String PT_NAACCR_RACE_CODE_PRIMARY, - String PT_NAACCR_RACE_CODE_SECONDARY, String PT_NAACCR_RACE_CODE_TERTIARY, String PT_NAACCR_ETHNICITY_CODE, - String PT_BIRTH_YEAR) { - this.DMP_ID_DEMO = ClinicalValueUtil.defaultWithNA(DMP_ID_DEMO); - this.PT_NAACCR_SEX_CODE = ClinicalValueUtil.defaultWithNA(PT_NAACCR_SEX_CODE); - this.PT_NAACCR_RACE_CODE_PRIMARY = ClinicalValueUtil.defaultWithNA(PT_NAACCR_RACE_CODE_PRIMARY); - this.PT_NAACCR_RACE_CODE_SECONDARY = ClinicalValueUtil.defaultWithNA(PT_NAACCR_RACE_CODE_SECONDARY); - this.PT_NAACCR_RACE_CODE_TERTIARY = ClinicalValueUtil.defaultWithNA(PT_NAACCR_RACE_CODE_TERTIARY); - this.PT_NAACCR_ETHNICITY_CODE = ClinicalValueUtil.defaultWithNA(PT_NAACCR_ETHNICITY_CODE); - this.PT_BIRTH_YEAR = ClinicalValueUtil.defaultWithNA(PT_BIRTH_YEAR); - } - - - public String getDMP_ID_DEMO() { - return DMP_ID_DEMO; - } - - public void setDMP_ID_DEMO(String DMP_ID_DEMO) { - this.DMP_ID_DEMO = ClinicalValueUtil.defaultWithNA(DMP_ID_DEMO); - } - - public String getPT_NAACCR_SEX_CODE() { - return PT_NAACCR_SEX_CODE; - } - - public void setPT_NAACCR_SEX_CODE(String PT_NAACCR_SEX_CODE) { - this.PT_NAACCR_SEX_CODE = ClinicalValueUtil.defaultWithNA(PT_NAACCR_SEX_CODE); - } - - public String getPT_NAACCR_RACE_CODE_PRIMARY() { - return PT_NAACCR_RACE_CODE_PRIMARY; - } - - public void setPT_NAACCR_RACE_CODE_PRIMARY(String PT_NAACCR_RACE_CODE_PRIMARY) { - this.PT_NAACCR_RACE_CODE_PRIMARY = ClinicalValueUtil.defaultWithNA(PT_NAACCR_RACE_CODE_PRIMARY); - } - - public String getPT_NAACCR_RACE_CODE_SECONDARY() { - return PT_NAACCR_RACE_CODE_SECONDARY; - } - - public void setPT_NAACCR_RACE_CODE_SECONDARY(String PT_NAACCR_RACE_CODE_SECONDARY) { - this.PT_NAACCR_RACE_CODE_SECONDARY = ClinicalValueUtil.defaultWithNA(PT_NAACCR_RACE_CODE_SECONDARY); - } - - public String getPT_NAACCR_RACE_CODE_TERTIARY() { - return PT_NAACCR_RACE_CODE_TERTIARY; - } - - public void setPT_NAACCR_RACE_CODE_TERTIARY(String PT_NAACCR_RACE_CODE_TERTIARY) { - this.PT_NAACCR_RACE_CODE_TERTIARY = ClinicalValueUtil.defaultWithNA(PT_NAACCR_RACE_CODE_TERTIARY); - } - - public String getPT_NAACCR_ETHNICITY_CODE() { - return PT_NAACCR_ETHNICITY_CODE; - } - - public void setPT_NAACCR_ETHNICITY_CODE(String PT_NAACCR_ETHNICITY_CODE) { - this.PT_NAACCR_ETHNICITY_CODE = ClinicalValueUtil.defaultWithNA(PT_NAACCR_ETHNICITY_CODE); - } - - public String getPT_BIRTH_YEAR() { - return PT_BIRTH_YEAR; - } - - public void setPT_BIRTH_YEAR(String PT_BIRTH_YEAR) { - this.PT_BIRTH_YEAR = ClinicalValueUtil.defaultWithNA(PT_BIRTH_YEAR); - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_ID_DEMO"); - fieldNames.add("PT_NAACCR_SEX_CODE"); - fieldNames.add("PT_NAACCR_RACE_CODE_PRIMARY"); - fieldNames.add("PT_NAACCR_RACE_CODE_SECONDARY"); - fieldNames.add("PT_NAACCR_RACE_CODE_TERTIARY"); - fieldNames.add("PT_NAACCR_ETHNICITY_CODE"); - fieldNames.add("PT_BIRTH_YEAR"); - return fieldNames; - } - - public List getHeaders() { - List headerNames = new ArrayList<>(); - headerNames.add("PATIENT_ID"); - headerNames.add("NAACCR_SEX_CODE"); - headerNames.add("NAACCR_RACE_CODE_PRIMARY"); - headerNames.add("NAACCR_RACE_CODE_SECONDARY"); - headerNames.add("NAACCR_RACE_CODE_TERTIARY"); - headerNames.add("NAACCR_ETHNICITY_CODE"); - headerNames.add("BIRTH_YEAR"); - return headerNames; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathology.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathology.java deleted file mode 100644 index f9ded6d36..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathology.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2016-2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class MskimpactPathology { - private String ptIdPathology; - private String dmpIdPathology; - private String ageAtPathologyReportInDays; - private String pathologyReportProcedureYear; - private String pathologyReportType; - private String pathologyReportYear; - private String ageAtPathologyReportDateInDays; - private String pathologyReportText; - - public MskimpactPathology() {} - - public String getPT_ID_PATHOLOGY() { - return ptIdPathology; - } - - public void setPT_ID_PATHOLOGY(String ptIdPathology) { - this.ptIdPathology = ptIdPathology; - } - - public String getDMP_ID_PATHOLOGY() { - return dmpIdPathology; - } - - public void setDMP_ID_PATHOLOGY(String dmpIdPathology) { - this.dmpIdPathology = dmpIdPathology; - } - - public String getAGE_AT_PATHOLOGY_REPORT_PROCEDURE_DATE_IN_DAYS() { - return ageAtPathologyReportInDays; - } - - public void setAGE_AT_PATHOLOGY_REPORT_PROCEDURE_DATE_IN_DAYS(String ageAtPathologyReportInDays) { - this.ageAtPathologyReportInDays = ageAtPathologyReportInDays; - } - - public String getPATHOLOGY_REPORT_PROCEDURE_YEAR() { - return pathologyReportProcedureYear; - } - - public void setPATHOLOGY_REPORT_PROCEDURE_YEAR(String pathologyReportProcedureYear) { - this.pathologyReportProcedureYear = pathologyReportProcedureYear; - } - - public String getPATHOLOGY_REPORT_TYPE() { - return pathologyReportType; - } - - public void setPATHOLOGY_REPORT_TYPE(String pathologyReportType) { - this.pathologyReportType = pathologyReportType; - } - - public String getPATHOLOGY_REPORT_YEAR() { - return pathologyReportYear; - } - - public void setPATHOLOGY_REPORT_YEAR(String pathologyReportYear) { - this.pathologyReportYear = pathologyReportYear; - } - - public String getAGE_AT_PATHOLOGY_REPORT_DATE_IN_DAYS() { - return ageAtPathologyReportDateInDays; - } - - public void setAGE_AT_PATHOLOGY_REPORT_DATE_IN_DAYS(String ageAtPathologyReportDateInDays) { - this.ageAtPathologyReportDateInDays = ageAtPathologyReportDateInDays; - } - - public String getPATHOLOGY_REPORT_TEXT() { - return pathologyReportText; - } - - public void setPATHOLOGY_REPORT_TEXT(String pathologyReportText) { - this.pathologyReportText = pathologyReportText; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathologyDmp.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathologyDmp.java deleted file mode 100644 index 935134a7c..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPathologyDmp.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2016-2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class MskimpactPathologyDmp { - private String ptIdPathDmp; - private String rptIdPathDmp; - private String dmpIdPathDmp; - private String sampleIdPathDmp; - - public MskimpactPathologyDmp() {} - - public String getPT_ID_PATH_DMP() { - return ptIdPathDmp; - } - - public void setPT_ID_PATH_DMP(String ptIdPathDmp) { - this.ptIdPathDmp = ptIdPathDmp; - } - - public String getRPT_ID_PATH_DMP() { - return rptIdPathDmp; - } - - public void setRPT_ID_PATH_DMP(String rptIdPathDmp) { - this.rptIdPathDmp = rptIdPathDmp; - } - - public String getDMP_ID_PATH_DMP() { - return dmpIdPathDmp; - } - - public void setDMP_ID_PATH_DMP(String dmpIdPathDmp) { - this.dmpIdPathDmp = dmpIdPathDmp; - } - - public String getSAMPLE_ID_PATH_DMP() { - return sampleIdPathDmp; - } - - public void setSAMPLE_ID_PATH_DMP(String sampleIdPathDmp) { - this.sampleIdPathDmp = sampleIdPathDmp; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientDemographics.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientDemographics.java deleted file mode 100644 index 69d34baac..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientDemographics.java +++ /dev/null @@ -1,510 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author jake - */ -public class MskimpactPatientDemographics { - - private final Integer currentYear = Calendar.getInstance().get(1); - private String PT_ID_DEMO; - private String DMP_ID_DEMO; - private String GENDER; - private String RACE; - private String ETHNICITY; - private String RELIGION; - private Integer AGE_AT_DATE_OF_DEATH_IN_DAYS; - private Integer AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS; - private Integer AGE_AT_TM_DX_DATE_IN_DAYS; - private String DEATH_SOURCE_DESCRIPTION; - private String PT_COUNTRY; - private String PT_STATE; - private String PT_ZIP3_CD; - private Integer PT_BIRTH_YEAR; - private String PT_SEX_DESC; - private String PT_VITAL_STATUS; - private String PT_MARITAL_STS_DESC; - private Integer PT_DEATH_YEAR; - private String PT_MRN_CREATE_YEAR; - private Integer TM_DX_YEAR; - private String OS_STATUS; - private String OS_MONTHS; - private String PT_NAACCR_ETHNICITY_CODE; - private String PT_NAACCR_RACE_CODE_PRIMARY; - private String PT_NAACCR_SEX_CODE; - private String PED_IND; - private String SAMPLE_ID_PATH_DMP; - private Integer LAST_CONTACT_YEAR; - private Integer AGE_AT_LAST_CONTACT_YEAR_IN_DAYS; - private Map additionalProperties = new HashMap<>(); - - public MskimpactPatientDemographics() {} - - public MskimpactPatientDemographics(String DMP_ID_DEMO, String PT_NAACCR_SEX_CODE, String PT_NAACCR_RACE_CODE_PRIMARY, String PT_NAACCR_ETHNICITY_CODE, String RELIGION, String PT_VITAL_STATUS, Integer PT_BIRTH_YEAR, Integer PT_DEATH_YEAR, Integer TM_DX_YEAR, Integer AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS, Integer AGE_AT_TM_DX_DATE_IN_DAYS, Integer AGE_AT_DATE_OF_DEATH_IN_DAYS, String PED_IND, String SAMPLE_ID_PATH_DMP, Integer LAST_CONTACT_YEAR, Integer AGE_AT_LAST_CONTACT_YEAR_IN_DAYS){ - this.DMP_ID_DEMO = ClinicalValueUtil.defaultWithNA(DMP_ID_DEMO); - this.PT_NAACCR_SEX_CODE = ClinicalValueUtil.defaultWithNegativeOne(PT_NAACCR_SEX_CODE); - this.PT_NAACCR_RACE_CODE_PRIMARY = ClinicalValueUtil.defaultWithNegativeOne(PT_NAACCR_RACE_CODE_PRIMARY); - this.PT_NAACCR_ETHNICITY_CODE = ClinicalValueUtil.defaultWithNegativeOne(PT_NAACCR_ETHNICITY_CODE); - this.RELIGION = ClinicalValueUtil.defaultWithNA(RELIGION); - this.PT_VITAL_STATUS = ClinicalValueUtil.defaultWithNA(PT_VITAL_STATUS); - this.TM_DX_YEAR = TM_DX_YEAR != null ? TM_DX_YEAR : -1; - this.PT_BIRTH_YEAR = PT_BIRTH_YEAR != null ? PT_BIRTH_YEAR : -1; - this.PT_DEATH_YEAR = PT_DEATH_YEAR != null ? PT_DEATH_YEAR : -1; - this.OS_STATUS = this.PT_VITAL_STATUS; - this.AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS = AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS; - this.AGE_AT_TM_DX_DATE_IN_DAYS = AGE_AT_TM_DX_DATE_IN_DAYS; - this.AGE_AT_DATE_OF_DEATH_IN_DAYS = AGE_AT_DATE_OF_DEATH_IN_DAYS; - this.PED_IND = PED_IND; - this.SAMPLE_ID_PATH_DMP = SAMPLE_ID_PATH_DMP; - this.LAST_CONTACT_YEAR = LAST_CONTACT_YEAR; - this.AGE_AT_LAST_CONTACT_YEAR_IN_DAYS = AGE_AT_LAST_CONTACT_YEAR_IN_DAYS; - } - - public MskimpactPatientDemographics(String DMP_ID_DEMO, Integer PT_BIRTH_YEAR, Integer PT_DEATH_YEAR) { - this.DMP_ID_DEMO = ClinicalValueUtil.defaultWithNA(DMP_ID_DEMO); - this.PT_BIRTH_YEAR = PT_BIRTH_YEAR != null ? PT_BIRTH_YEAR : -1; - this.PT_DEATH_YEAR = PT_DEATH_YEAR != null ? PT_DEATH_YEAR : -1; - } - - public Integer getTM_DX_YEAR() { - return TM_DX_YEAR; - } - - public void setTM_DX_YEAR(Integer TM_DX_YEAR) { - this.TM_DX_YEAR = TM_DX_YEAR != null ? TM_DX_YEAR : -1; - } - - public String getAGE_AT_DIAGNOSIS(){ - if(this.PT_BIRTH_YEAR>-1 && this.TM_DX_YEAR>-1 && this.TM_DX_YEAR>this.PT_BIRTH_YEAR){ - Integer i = this.TM_DX_YEAR - this.PT_BIRTH_YEAR; - //Age > 90 is considered identifying - if(i<90){ - return i.toString(); - } - else{ - return "NA"; - } - } - else{ - return "NA"; - } - } - - public String getPT_ID_DEMO() { - return PT_ID_DEMO; - } - - public void setPT_ID_DEMO(String PT_ID_DEMO) { - this.PT_ID_DEMO = ClinicalValueUtil.defaultWithNA(PT_ID_DEMO); - } - - public String getDMP_ID_DEMO() { - return DMP_ID_DEMO; - } - - public void setDMP_ID_DEMO(String DMP_ID_DEMO) { - this.DMP_ID_DEMO = ClinicalValueUtil.defaultWithNA(DMP_ID_DEMO); - } - - public String getGENDER() { - return GENDER; - } - - public void setGENDER(String GENDER) { - this.GENDER = ClinicalValueUtil.defaultWithNA(GENDER); - } - - public String getRACE() { - return RACE; - } - - public void setRACE(String RACE) { - this.RACE = ClinicalValueUtil.defaultWithNA(RACE); - } - - public String getRELIGION() { - return RELIGION; - } - - public void setRELIGION(String RELIGION) { - this.RELIGION = ClinicalValueUtil.defaultWithNA(RELIGION); - } - - public Integer getAGE_AT_DATE_OF_DEATH_IN_DAYS() { - return AGE_AT_DATE_OF_DEATH_IN_DAYS; - } - - public void setAGE_AT_DATE_OF_DEATH_IN_DAYS(Integer AGE_AT_DATE_OF_DEATH_IN_DAYS) { - this.AGE_AT_DATE_OF_DEATH_IN_DAYS = AGE_AT_DATE_OF_DEATH_IN_DAYS != null ? AGE_AT_DATE_OF_DEATH_IN_DAYS : -1; - } - - public Integer getAGE_AT_LAST_KNOWN_ALIVE_IN_DAYS() { - return AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS; - } - - public void setAGE_AT_LAST_KNOWN_ALIVE_IN_DAYS(Integer AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS) { - this.AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS = AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS != null ? AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS : -1; - } - - public Integer getAGE_AT_TM_DX_DATE_IN_DAYS() { - return AGE_AT_DATE_OF_DEATH_IN_DAYS; - } - - public void setAGE_AT_TM_DX_DATE_IN_DAYS(Integer AGE_AT_TM_DX_DATE_IN_DAYS) { - this.AGE_AT_TM_DX_DATE_IN_DAYS = AGE_AT_TM_DX_DATE_IN_DAYS != null ? AGE_AT_TM_DX_DATE_IN_DAYS : -1; - } - - public String getDEATH_SOURCE_DESCRIPTION() { - return DEATH_SOURCE_DESCRIPTION; - } - - public void setDEATH_SOURCE_DESCRIPTION(String DEATH_SOURCE_DESCRIPTION) { - this.DEATH_SOURCE_DESCRIPTION = ClinicalValueUtil.defaultWithNA(DEATH_SOURCE_DESCRIPTION); - } - - public String getPT_COUNTRY() { - return PT_COUNTRY; - } - - public void setPT_COUNTRY(String PT_COUNTRY) { - this.PT_COUNTRY = ClinicalValueUtil.defaultWithNA(PT_COUNTRY); - } - - public String getPT_STATE() { - return PT_STATE; - } - - public void setPT_STATE(String PT_STATE) { - this.PT_STATE = ClinicalValueUtil.defaultWithNA(PT_STATE); - } - - public String getPT_ZIP3_CD() { - return PT_ZIP3_CD; - } - - public void setPT_ZIP3_CD(String PT_ZIP3_CD) { - this.PT_ZIP3_CD = ClinicalValueUtil.defaultWithNA(PT_ZIP3_CD); - } - - public Integer getPT_BIRTH_YEAR() { - return PT_BIRTH_YEAR; - } - - public void setPT_BIRTH_YEAR(Integer PT_BIRTH_YEAR) { - this.PT_BIRTH_YEAR = PT_BIRTH_YEAR != null ? PT_BIRTH_YEAR : -1; - } - - public String getPT_SEX_DESC() { - return PT_SEX_DESC; - } - - public void setPT_SEX_DESC(String PT_SEX_DESC) { - this.PT_SEX_DESC = ClinicalValueUtil.defaultWithNA(PT_SEX_DESC); - } - - public String getPT_VITAL_STATUS() { - return PT_VITAL_STATUS; - } - - public void setPT_VITAL_STATUS(String PT_VITAL_STATUS) { - this.PT_VITAL_STATUS = ClinicalValueUtil.defaultWithNA(PT_VITAL_STATUS); - } - - public String getPT_MARITAL_STS_DESC() { - return PT_MARITAL_STS_DESC; - } - - public void setPT_MARITAL_STS_DESC(String PT_MARITAL_STS_DESC) { - this.PT_MARITAL_STS_DESC = ClinicalValueUtil.defaultWithNA(PT_MARITAL_STS_DESC); - } - - public Integer getPT_DEATH_YEAR() { - return PT_DEATH_YEAR; - } - - public void setPT_DEATH_YEAR(Integer PT_DEATH_YEAR) { - this.PT_DEATH_YEAR = PT_DEATH_YEAR != null ? PT_DEATH_YEAR : -1; - } - - public String getPT_MRN_CREATE_YEAR() { - return PT_MRN_CREATE_YEAR; - } - - public void setPT_MRN_CREATE_YEAR(String PT_MRN_CREATE_YEAR) { - this.PT_MRN_CREATE_YEAR = ClinicalValueUtil.defaultWithNA(PT_MRN_CREATE_YEAR); - } - - public String getOS_STATUS(){ - return OS_STATUS.trim().equals("ALIVE") ? "LIVING" : "DECEASED"; - } - - public void setOS_STATUS(String OS_STATUS) { - this.OS_STATUS = ClinicalValueUtil.defaultWithNA(OS_STATUS); - } - - public String getOS_MONTHS() { - if (AGE_AT_TM_DX_DATE_IN_DAYS != null && AGE_AT_TM_DX_DATE_IN_DAYS < 0) { - // can't calculate OS_MONTHS properly, return NA - return "NA"; - } - if (getOS_STATUS().equals("LIVING")) { - if (AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS != null && AGE_AT_TM_DX_DATE_IN_DAYS != null) { - return String.format("%.3f", (AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS - AGE_AT_TM_DX_DATE_IN_DAYS) / 30.4167); - } - } - else { - if (AGE_AT_DATE_OF_DEATH_IN_DAYS != null && AGE_AT_TM_DX_DATE_IN_DAYS != null) { - return String.format("%.3f", (AGE_AT_DATE_OF_DEATH_IN_DAYS - AGE_AT_TM_DX_DATE_IN_DAYS) / 30.4167); - } - } - return "NA"; - } - - public void setOS_MONTHS() { - this.OS_MONTHS = ClinicalValueUtil.defaultWithNA(OS_MONTHS); - } - - public String getDARWIN_PATIENT_AGE(){ - if(this.PT_BIRTH_YEAR>-1){ - if(this.PT_DEATH_YEAR>-1){ - Integer i = this.PT_DEATH_YEAR-this.PT_BIRTH_YEAR; - //Age > 90 is considered identifying - if (i > 90){ - return ">90"; - } - return i.toString(); - } - Integer i = currentYear-this.PT_BIRTH_YEAR; - //Age > 90 is considered identifying - if (i > 90) { - return ">90"; - } - return i.toString(); - } - else{ - return "NA"; - } - } - - public String getYearsSinceBirth() { - if (PT_BIRTH_YEAR > -1) { - return String.valueOf(currentYear - PT_BIRTH_YEAR); - } - return "NA"; - } - - public void setPED_IND(String PED_IND) { - this.PED_IND = PED_IND; - } - - public String getPED_IND() { - if (PED_IND.equals("Y")) { - return "Yes"; - } - else if (PED_IND.equals("N")) { - return "No"; - } - else { - return PED_IND; - } - } - - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - - public static List getPatientDemographicsFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_ID_DEMO"); - fieldNames.add("DARWIN_PATIENT_AGE"); - fieldNames.add("RACE"); - fieldNames.add("RELIGION"); - fieldNames.add("GENDER"); - fieldNames.add("ETHNICITY"); - fieldNames.add("OS_STATUS"); - fieldNames.add("OS_MONTHS"); - fieldNames.add("PED_IND"); - return fieldNames; - } - - public static List getPatientDemographicsHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("AGE_CURRENT"); // DARWIN_PATIENT_AGE has been renamed - fieldNames.add("RACE"); - fieldNames.add("RELIGION"); - fieldNames.add("SEX"); - fieldNames.add("ETHNICITY"); - fieldNames.add("OS_STATUS"); - fieldNames.add("OS_MONTHS"); - fieldNames.add("PED_IND"); - return fieldNames; - } - - public static List getAgeFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_ID_DEMO"); - fieldNames.add("YearsSinceBirth"); - return fieldNames; - } - - public static List getAgeHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("AGE"); - return fieldNames; - } - - public static List getVitalStatusFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("DMP_ID_DEMO"); - fieldNames.add("LAST_CONTACT_YEAR"); - fieldNames.add("PT_DEATH_YEAR"); - fieldNames.add("AGE_AT_LAST_CONTACT_YEAR_IN_DAYS"); - fieldNames.add("AGE_AT_DATE_OF_DEATH_IN_DAYS"); - fieldNames.add("OS_STATUS"); - return fieldNames; - } - - public static List getVitalStatusHeaders() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("YEAR_CONTACT"); - fieldNames.add("YEAR_DEATH"); - fieldNames.add("INT_CONTACT"); - fieldNames.add("INT_DOD"); - fieldNames.add("DEAD"); - return fieldNames; - } - - public void setAdditionalProperty(String name, Object value) { - this.additionalProperties.put(name, value); - } - - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - /** - * @return the PT_NAACCR_ETHNICITY_CODE - */ - public String getPT_NAACCR_ETHNICITY_CODE() { - return PT_NAACCR_ETHNICITY_CODE; - } - - /** - * @param PT_NAACCR_ETHNICITY_CODE the PT_NAACCR_ETHNICITY_CODE to set - */ - public void setPT_NAACCR_ETHNICITY_CODE(String PT_NAACCR_ETHNICITY_CODE) { - this.PT_NAACCR_ETHNICITY_CODE = PT_NAACCR_ETHNICITY_CODE; - } - - /** - * @return the PT_NAACCR_RACE_CODE_PRIMARY - */ - public String getPT_NAACCR_RACE_CODE_PRIMARY() { - return PT_NAACCR_RACE_CODE_PRIMARY; - } - - /** - * @param PT_NAACCR_RACE_CODE_PRIMARY the NAACCR_RACE_CODE to set - */ - public void setPT_NAACCR_RACE_CODE_PRIMARY(String PT_NAACCR_RACE_CODE_PRIMARY) { - this.PT_NAACCR_RACE_CODE_PRIMARY = PT_NAACCR_RACE_CODE_PRIMARY; - } - - /** - * @return the PT_NAACCR_SEX_CODE - */ - public String getPT_NAACCR_SEX_CODE() { - return PT_NAACCR_SEX_CODE; - } - - /** - * @param PT_NAACCR_SEX_CODE the PT_NAACCR_SEX_CODE to set - */ - public void setPT_NAACCR_SEX_CODE(String PT_NAACCR_SEX_CODE) { - this.PT_NAACCR_SEX_CODE = PT_NAACCR_SEX_CODE; - } - - /** - * @return the ETHNICITY - */ - public String getETHNICITY() { - return ETHNICITY; - } - - /** - * @param ETHNICITY the ETHNICITY to set - */ - public void setETHNICITY(String ETHNICITY) { - this.ETHNICITY = ETHNICITY; - } - - /** - * @return the SAMPLE_ID_PATH_DMP - */ - public String getSAMPLE_ID_PATH_DMP() { - return SAMPLE_ID_PATH_DMP; - } - - /** - * @param SAMPLE_ID_PATH_DMP the SAMPLE_ID_PATH_DMP to set - */ - public void setSAMPLE_ID_PATH_DMP(String SAMPLE_ID_PATH_DMP) { - this.SAMPLE_ID_PATH_DMP = SAMPLE_ID_PATH_DMP; - } - - public Integer getLAST_CONTACT_YEAR() { - return LAST_CONTACT_YEAR; - } - - public void setLAST_CONTACT_YEAR(Integer LAST_CONTACT_YEAR) { - this.LAST_CONTACT_YEAR = LAST_CONTACT_YEAR; - } - - public Integer getAGE_AT_LAST_CONTACT_YEAR_IN_DAYS() { - return AGE_AT_LAST_CONTACT_YEAR_IN_DAYS; - } - - public void setAGE_AT_LAST_CONTACT_YEAR_IN_DAYS(Integer AGE_AT_LAST_CONTACT_YEAR_IN_DAYS) { - this.AGE_AT_LAST_CONTACT_YEAR_IN_DAYS = AGE_AT_LAST_CONTACT_YEAR_IN_DAYS; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientIcdoRecord.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientIcdoRecord.java deleted file mode 100644 index ef05282bd..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/MskimpactPatientIcdoRecord.java +++ /dev/null @@ -1,2213 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author jake - */ -public class MskimpactPatientIcdoRecord { - private Integer TUMOR_YEAR; - private String PT_ID_ICDO; - private String DMP_ID_ICDO; - private String TM_TUMOR_SEQ_DESC; - private String TM_ACC_YEAR; - private String TM_FIRST_MSK_YEAR; - private String AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS; - private String TM_CASE_STS; - private String TM_CASE_STS_DESC; - private String TM_CASE_EFF_YEAR; - private String AGE_AT_TM_CASE_EFF_DATE_IN_DAYS; - private String TM_STATE_AT_DX; - private String TM_SMOKING_HX; - private String TM_SMOKING_HX_DESC; - private String TM_OCCUPATION; - private String TM_OCCUPATION_DESC; - private String TM_FACILITY_FROM; - private String FACILITY_FROM_DESC; - private String TM_FACILITY_TO; - private String FACILITY_TO_DESC; - private Integer TM_DX_YEAR; - private Integer AGE_AT_TM_DX_DATE_IN_DAYS; - private String TM_SITE_CD; - private String TM_SITE_DESC; - private String TM_LATERALITY_CD; - private String TM_LATERALITY_DESC; - private String TM_HIST_CD; - private String TM_HIST_DESC; - private String TM_DX_CONFRM_CD; - private String TM_DX_CONFRM_DESC; - private String TM_REGNODE_EXM_NO; - private String TM_REGNODE_POS_NO; - private String TM_TUMOR_SIZE; - private String TM_RESID_TUMOR_CD; - private String TM_RESID_TUMOR_DESC; - private String TM_GENERAL_STG; - private String TM_GENERAL_STG_DESC; - private String TM_TNM_EDITION; - private String TM_TNM_EDITION_DESC; - private String TM_CLIN_TNM_T; - private String TM_CLIN_TNM_T_DESC; - private String TM_CLIN_TNM_N; - private String TM_CLIN_TNM_N_DESC; - private String TM_CLIN_TNM_M; - private String TM_CLIN_TNM_M_DESC; - private String TM_CLIN_STG_GRP; - private String TM_PATH_TNM_T; - private String TM_PATH_TNM_T_DESC; - private String TM_PATH_TNM_N; - private String TM_PATH_TNM_N_DESC; - private String TM_PATH_TNM_M; - private String TM_PATH_TNM_M_DESC; - private String TM_PATH_STG_GRP; - private String TM_PATH_RPT_AV; - private String TM_CA_STS_AT_ACC; - private String TM_FIRST_RECUR_YEAR; - private String AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS; - private String TM_FIRST_RECUR_TYP; - private String TM_ADM_YEAR; - private String AGE_AT_TM_ADM_DATE_IN_DAYS; - private String TM_DSCH_YEAR; - private String AGE_AT_TM_DSCH_DATE_IN_DAYS; - private String TM_SURG_DSCH_YEAR; - private String AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS; - private String TM_READM_WTHN_30D; - private String TM_FIRST_TX_YEAR; - private String AGE_AT_TM_FIRST_TX_DATE_IN_DAYS; - private String TM_NON_CA_SURG_SUM; - private String TM_NON_CA_SURG_SUM_DESC; - private String TM_NON_CA_SURG_MSK; - private String TM_NON_CA_SURG_YEAR; - private String AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS; - private String TM_CA_SURG_98; - private String TM_CA_SURG_98_MSK; - private String TM_CA_SURG_03; - private String TM_CA_SURG_03_MSK; - private String TM_OTH_SURG_98; - private String TM_OTH_SURG_98_MSK; - private String TM_OTH_SURG_03; - private String TM_OTH_SURG_03_MSK; - private String TM_OTH_SURG_CD; - private String TM_OTH_SURG_CD_DESC; - private String TM_RGN_SCOP_98; - private String TM_RGN_SCOP_98_MSK; - private String TM_RGN_SCOP_03; - private String TM_RGN_SCOP_03_MSK; - private String TM_REGNODE_SCOP_CD; - private String TM_REGNODE_SCOP_DESC; - private String TM_RECON_SURG; - private String TM_RECON_SURG_DESC; - private String TM_CA_SURG_YEAR; - private String AGE_AT_TM_CA_SURG_DATE_IN_DAYS; - private String TM_SURG_DEF_YEAR; - private String AGE_AT_TM_SURG_DEF_DATE_IN_DAYS; - private String TM_REASON_NO_SURG; - private String REASON_NO_SURG_DESC; - private String TM_PRIM_SURGEON; - private String TM_PRIM_SURGEON_NAME; - private String TM_ATN_DR_NO; - private String TM_ATN_DR_NAME; - private String TM_REASON_NO_RAD; - private String TM_REASON_NO_RAD_DESC; - private String TM_RAD_STRT_YEAR; - private String AGE_AT_TM_RAD_STRT_DATE_IN_DAYS; - private String TM_RAD_END_YEAR; - private String AGE_AT_TM_RAD_END_DATE_IN_DAYS; - private String TM_RAD_TX_MOD; - private String TM_RAD_TX_MOD_DESC; - private String TM_BOOST_RAD_MOD; - private String TM_BOOST_RAD_MOD_DESC; - private String TM_LOC_RAD_TX; - private String TM_LOC_RAD_TX_DESC; - private String TM_RAD_TX_VOL; - private String TM_RAD_TX_VOL_DESC; - private String TM_NUM_TX_THIS_VOL; - private String TM_NUM_TX_THIS_VOL_DESC; - private String TM_REG_RAD_DOSE; - private String TM_REG_RAD_DOSE_DESC; - private String TM_BOOST_RAD_DOSE; - private String TM_BOOST_RAD_DOSE_DESC; - private String TM_RAD_SURG_SEQ; - private String TM_RAD_SURG_SEQ_DESC; - private String TM_RAD_MD; - private String TM_RAD_MD_NAME; - private String TM_SYST_STRT_YEAR; - private String AGE_AT_TM_SYST_STRT_DATE_IN_DAYS; - private String TM_OTH_STRT_YEAR; - private String AGE_AT_TM_OTH_STRT_DATE_IN_DAYS; - private String TM_CHEM_SUM; - private String TM_CHEM_SUM_DESC; - private String TM_CHEM_SUM_MSK; - private String TM_CHEM_SUM_MSK_DESC; - private String TM_TUMOR_SEQ; - private String TM_HORM_SUM; - private String TM_HORM_SUM_DESC; - private String TM_HORM_SUM_MSK; - private String TM_HORM_SUM_MSK_DESC; - private String TM_BRM_SUM; - private String TM_BRM_SUM_DESC; - private String TM_BRM_SUM_MSK; - private String TM_BRM_SUM_MSK_DESC; - private String TM_OTH_SUM; - private String TM_OTH_SUM_DESC; - private String TM_OTH_SUM_MSK; - private String TM_OTH_SUM_MSK_DESC; - private String TM_PALLIA_PROC; - private String TM_PALLIA_PROC_DESC; - private String TM_PALLIA_PROC_MSK; - private String TM_PALLIA_PROC_MSK_DESC; - private String TM_ONCOLOGY_MD; - private String TM_ONCOLOGY_MD_NAME; - private String TM_PRCS_YEAR; - private String AGE_AT_TM_PRCS_DATE_IN_DAYS; - private String TM_PATH_TEXT; - private String TM_SURG_TEXT; - private String TM_OVERRIDE_COM; - private String TM_CSSIZE; - private String TM_CSEXT; - private String TM_CSEXTEV; - private String TM_CSLMND; - private String TM_CSRGNEV; - private String TM_CSMETDX; - private String TM_CSMETEV; - private String TM_TSTAGE; - private String TM_TSTAGE_DESC; - private String TM_NSTAGE; - private String TM_NSTAGE_DESC; - private String TM_MSTAGE; - private String TM_MSTAGE_DESC; - private String TM_TBASIS; - private String TM_TBASIS_DESC; - private String TM_NBASIS; - private String TM_NBASIS_DESC; - private String TM_MBASIS; - private String TM_MBASIS_DESC; - private String TM_AJCC; - private String TM_AJCC_DESC; - private String TM_MSK_STG; - private Integer AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS; - private Integer AGE_AT_DATE_OF_DEATH_IN_DAYS; - - public MskimpactPatientIcdoRecord(){} - - public MskimpactPatientIcdoRecord( - Integer TUMOR_YEAR, - String PT_ID_ICDO, - String DMP_ID_ICDO, - String TM_TUMOR_SEQ_DESC, - String TM_ACC_YEAR, - String TM_FIRST_MSK_YEAR, - String AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS, - String TM_CASE_STS, - String TM_CASE_STS_DESC, - String TM_CASE_EFF_YEAR, - String AGE_AT_TM_CASE_EFF_DATE_IN_DAYS, - String TM_STATE_AT_DX, - String TM_SMOKING_HX, - String TM_SMOKING_HX_DESC, - String TM_OCCUPATION, - String TM_OCCUPATION_DESC, - String TM_FACILITY_FROM, - String FACILITY_FROM_DESC, - String TM_FACILITY_TO, - String FACILITY_TO_DESC, - Integer TM_DX_YEAR, - Integer AGE_AT_TM_DX_DATE_IN_DAYS, - String TM_SITE_CD, - String TM_SITE_DESC, - String TM_LATERALITY_CD, - String TM_LATERALITY_DESC, - String TM_HIST_CD, - String TM_HIST_DESC, - String TM_DX_CONFRM_CD, - String TM_DX_CONFRM_DESC, - String TM_REGNODE_EXM_NO, - String TM_REGNODE_POS_NO, - String TM_TUMOR_SIZE, - String TM_RESID_TUMOR_CD, - String TM_RESID_TUMOR_DESC, - String TM_GENERAL_STG, - String TM_GENERAL_STG_DESC, - String TM_TNM_EDITION, - String TM_TNM_EDITION_DESC, - String TM_CLIN_TNM_T, - String TM_CLIN_TNM_T_DESC, - String TM_CLIN_TNM_N, - String TM_CLIN_TNM_N_DESC, - String TM_CLIN_TNM_M, - String TM_CLIN_TNM_M_DESC, - String TM_CLIN_STG_GRP, - String TM_PATH_TNM_T, - String TM_PATH_TNM_T_DESC, - String TM_PATH_TNM_N, - String TM_PATH_TNM_N_DESC, - String TM_PATH_TNM_M, - String TM_PATH_TNM_M_DESC, - String TM_PATH_STG_GRP, - String TM_PATH_RPT_AV, - String TM_CA_STS_AT_ACC, - String TM_FIRST_RECUR_YEAR, - String AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS, - String TM_FIRST_RECUR_TYP, - String TM_ADM_YEAR, - String AGE_AT_TM_ADM_DATE_IN_DAYS, - String TM_DSCH_YEAR, - String AGE_AT_TM_DSCH_DATE_IN_DAYS, - String TM_SURG_DSCH_YEAR, - String AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS, - String TM_READM_WTHN_30D, - String TM_FIRST_TX_YEAR, - String AGE_AT_TM_FIRST_TX_DATE_IN_DAYS, - String TM_NON_CA_SURG_SUM, - String TM_NON_CA_SURG_SUM_DESC, - String TM_NON_CA_SURG_MSK, - String TM_NON_CA_SURG_YEAR, - String AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS, - String TM_CA_SURG_98, - String TM_CA_SURG_98_MSK, - String TM_CA_SURG_03, - String TM_CA_SURG_03_MSK, - String TM_OTH_SURG_98, - String TM_OTH_SURG_98_MSK, - String TM_OTH_SURG_03, - String TM_OTH_SURG_03_MSK, - String TM_OTH_SURG_CD, - String TM_OTH_SURG_CD_DESC, - String TM_RGN_SCOP_98, - String TM_RGN_SCOP_98_MSK, - String TM_RGN_SCOP_03, - String TM_RGN_SCOP_03_MSK, - String TM_REGNODE_SCOP_CD, - String TM_REGNODE_SCOP_DESC, - String TM_RECON_SURG, - String TM_RECON_SURG_DESC, - String TM_CA_SURG_YEAR, - String AGE_AT_TM_CA_SURG_DATE_IN_DAYS, - String TM_SURG_DEF_YEAR, - String AGE_AT_TM_SURG_DEF_DATE_IN_DAYS, - String TM_REASON_NO_SURG, - String REASON_NO_SURG_DESC, - String TM_PRIM_SURGEON, - String TM_PRIM_SURGEON_NAME, - String TM_ATN_DR_NO, - String TM_ATN_DR_NAME, - String TM_REASON_NO_RAD, - String TM_REASON_NO_RAD_DESC, - String TM_RAD_STRT_YEAR, - String AGE_AT_TM_RAD_STRT_DATE_IN_DAYS, - String TM_RAD_END_YEAR, - String AGE_AT_TM_RAD_END_DATE_IN_DAYS, - String TM_RAD_TX_MOD, - String TM_RAD_TX_MOD_DESC, - String TM_BOOST_RAD_MOD, - String TM_BOOST_RAD_MOD_DESC, - String TM_LOC_RAD_TX, - String TM_LOC_RAD_TX_DESC, - String TM_RAD_TX_VOL, - String TM_RAD_TX_VOL_DESC, - String TM_NUM_TX_THIS_VOL, - String TM_NUM_TX_THIS_VOL_DESC, - String TM_REG_RAD_DOSE, - String TM_REG_RAD_DOSE_DESC, - String TM_BOOST_RAD_DOSE, - String TM_BOOST_RAD_DOSE_DESC, - String TM_RAD_SURG_SEQ, - String TM_RAD_SURG_SEQ_DESC, - String TM_RAD_MD, - String TM_RAD_MD_NAME, - String TM_SYST_STRT_YEAR, - String AGE_AT_TM_SYST_STRT_DATE_IN_DAYS, - String TM_OTH_STRT_YEAR, - String AGE_AT_TM_OTH_STRT_DATE_IN_DAYS, - String TM_CHEM_SUM, - String TM_CHEM_SUM_DESC, - String TM_CHEM_SUM_MSK, - String TM_CHEM_SUM_MSK_DESC, - String TM_TUMOR_SEQ, - String TM_HORM_SUM, - String TM_HORM_SUM_DESC, - String TM_HORM_SUM_MSK, - String TM_HORM_SUM_MSK_DESC, - String TM_BRM_SUM, - String TM_BRM_SUM_DESC, - String TM_BRM_SUM_MSK, - String TM_BRM_SUM_MSK_DESC, - String TM_OTH_SUM, - String TM_OTH_SUM_DESC, - String TM_OTH_SUM_MSK, - String TM_OTH_SUM_MSK_DESC, - String TM_PALLIA_PROC, - String TM_PALLIA_PROC_DESC, - String TM_PALLIA_PROC_MSK, - String TM_PALLIA_PROC_MSK_DESC, - String TM_ONCOLOGY_MD, - String TM_ONCOLOGY_MD_NAME, - String TM_PRCS_YEAR, - String AGE_AT_TM_PRCS_DATE_IN_DAYS, - String TM_PATH_TEXT, - String TM_SURG_TEXT, - String TM_OVERRIDE_COM, - String TM_CSSIZE, - String TM_CSEXT, - String TM_CSEXTEV, - String TM_CSLMND, - String TM_CSRGNEV, - String TM_CSMETDX, - String TM_CSMETEV, - String TM_TSTAGE, - String TM_TSTAGE_DESC, - String TM_NSTAGE, - String TM_NSTAGE_DESC, - String TM_MSTAGE, - String TM_MSTAGE_DESC, - String TM_TBASIS, - String TM_TBASIS_DESC, - String TM_NBASIS, - String TM_NBASIS_DESC, - String TM_MBASIS, - String TM_MBASIS_DESC, - String TM_AJCC, - String TM_AJCC_DESC, - String TM_MSK_STG){ - this.TUMOR_YEAR = TUMOR_YEAR != null ? TUMOR_YEAR : -1; - this.PT_ID_ICDO = ClinicalValueUtil.defaultWithNA(PT_ID_ICDO); - this.DMP_ID_ICDO = ClinicalValueUtil.defaultWithNA(DMP_ID_ICDO); - this.TM_TUMOR_SEQ_DESC = ClinicalValueUtil.defaultWithNA(TM_TUMOR_SEQ_DESC); - this.TM_ACC_YEAR = ClinicalValueUtil.defaultWithNA(TM_ACC_YEAR); - this.TM_FIRST_MSK_YEAR = ClinicalValueUtil.defaultWithNA(TM_FIRST_MSK_YEAR); - this.AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS); - this.TM_CASE_STS = ClinicalValueUtil.defaultWithNA(TM_CASE_STS); - this.TM_CASE_STS_DESC = ClinicalValueUtil.defaultWithNA(TM_CASE_STS_DESC); - this.TM_CASE_EFF_YEAR = ClinicalValueUtil.defaultWithNA(TM_CASE_EFF_YEAR); - this.AGE_AT_TM_CASE_EFF_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_CASE_EFF_DATE_IN_DAYS); - this.TM_STATE_AT_DX = ClinicalValueUtil.defaultWithNA(TM_STATE_AT_DX); - this.TM_SMOKING_HX = ClinicalValueUtil.defaultWithNA(TM_SMOKING_HX); - this.TM_SMOKING_HX_DESC = ClinicalValueUtil.defaultWithNA(TM_SMOKING_HX_DESC); - this.TM_OCCUPATION = ClinicalValueUtil.defaultWithNA(TM_OCCUPATION); - this.TM_OCCUPATION_DESC = ClinicalValueUtil.defaultWithNA(TM_OCCUPATION_DESC); - this.TM_FACILITY_FROM = ClinicalValueUtil.defaultWithNA(TM_FACILITY_FROM); - this.FACILITY_FROM_DESC = ClinicalValueUtil.defaultWithNA(FACILITY_FROM_DESC); - this.TM_FACILITY_TO = ClinicalValueUtil.defaultWithNA(TM_FACILITY_TO); - this.FACILITY_TO_DESC = ClinicalValueUtil.defaultWithNA(FACILITY_TO_DESC); - this.TM_DX_YEAR = TM_DX_YEAR != null ? TM_DX_YEAR : 0; - this.AGE_AT_TM_DX_DATE_IN_DAYS = AGE_AT_TM_DX_DATE_IN_DAYS != null ? AGE_AT_TM_DX_DATE_IN_DAYS : -1; - this.TM_SITE_CD = ClinicalValueUtil.defaultWithNA(TM_SITE_CD); - this.TM_SITE_DESC = ClinicalValueUtil.defaultWithNA(TM_SITE_DESC); - this.TM_LATERALITY_CD = ClinicalValueUtil.defaultWithNA(TM_LATERALITY_CD); - this.TM_LATERALITY_DESC = ClinicalValueUtil.defaultWithNA(TM_LATERALITY_DESC); - this.TM_HIST_CD = ClinicalValueUtil.defaultWithNA(TM_HIST_CD); - this.TM_HIST_DESC = ClinicalValueUtil.defaultWithNA(TM_HIST_DESC); - this.TM_DX_CONFRM_CD = ClinicalValueUtil.defaultWithNA(TM_DX_CONFRM_CD); - this.TM_DX_CONFRM_DESC = ClinicalValueUtil.defaultWithNA(TM_DX_CONFRM_DESC); - this.TM_REGNODE_EXM_NO = ClinicalValueUtil.defaultWithNA(TM_REGNODE_EXM_NO); - this.TM_REGNODE_POS_NO = ClinicalValueUtil.defaultWithNA(TM_REGNODE_POS_NO); - this.TM_TUMOR_SIZE = ClinicalValueUtil.defaultWithNA(TM_TUMOR_SIZE); - this.TM_RESID_TUMOR_CD = ClinicalValueUtil.defaultWithNA(TM_RESID_TUMOR_CD); - this.TM_RESID_TUMOR_DESC = ClinicalValueUtil.defaultWithNA(TM_RESID_TUMOR_DESC); - this.TM_GENERAL_STG = ClinicalValueUtil.defaultWithNA(TM_GENERAL_STG); - this.TM_GENERAL_STG_DESC = ClinicalValueUtil.defaultWithNA(TM_GENERAL_STG_DESC); - this.TM_TNM_EDITION = ClinicalValueUtil.defaultWithNA(TM_TNM_EDITION); - this.TM_TNM_EDITION_DESC = ClinicalValueUtil.defaultWithNA(TM_TNM_EDITION_DESC); - this.TM_CLIN_TNM_T = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_T); - this.TM_CLIN_TNM_T_DESC = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_T_DESC); - this.TM_CLIN_TNM_N = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_N); - this.TM_CLIN_TNM_N_DESC = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_N_DESC); - this.TM_CLIN_TNM_M = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_M); - this.TM_CLIN_TNM_M_DESC = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_M_DESC); - this.TM_CLIN_STG_GRP = ClinicalValueUtil.defaultWithNA(TM_CLIN_STG_GRP); - this.TM_PATH_TNM_T = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_T); - this.TM_PATH_TNM_T_DESC = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_T_DESC); - this.TM_PATH_TNM_N = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_N); - this.TM_PATH_TNM_N_DESC = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_N_DESC); - this.TM_PATH_TNM_M = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_M); - this.TM_PATH_TNM_M_DESC = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_M_DESC); - this.TM_PATH_STG_GRP = ClinicalValueUtil.defaultWithNA(TM_PATH_STG_GRP); - this.TM_PATH_RPT_AV = ClinicalValueUtil.defaultWithNA(TM_PATH_RPT_AV); - this.TM_CA_STS_AT_ACC = ClinicalValueUtil.defaultWithNA(TM_CA_STS_AT_ACC); - this.TM_FIRST_RECUR_YEAR = ClinicalValueUtil.defaultWithNA(TM_FIRST_RECUR_YEAR); - this.AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS); - this.TM_FIRST_RECUR_TYP = ClinicalValueUtil.defaultWithNA(TM_FIRST_RECUR_TYP); - this.TM_ADM_YEAR = ClinicalValueUtil.defaultWithNA(TM_ADM_YEAR); - this.AGE_AT_TM_ADM_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_ADM_DATE_IN_DAYS); - this.TM_DSCH_YEAR = ClinicalValueUtil.defaultWithNA(TM_DSCH_YEAR); - this.AGE_AT_TM_DSCH_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_DSCH_DATE_IN_DAYS); - this.TM_SURG_DSCH_YEAR = ClinicalValueUtil.defaultWithNA(TM_SURG_DSCH_YEAR); - this.AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS); - this.TM_READM_WTHN_30D = ClinicalValueUtil.defaultWithNA(TM_READM_WTHN_30D); - this.TM_FIRST_TX_YEAR = ClinicalValueUtil.defaultWithNA(TM_FIRST_TX_YEAR); - this.AGE_AT_TM_FIRST_TX_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_FIRST_TX_DATE_IN_DAYS); - this.TM_NON_CA_SURG_SUM = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_SUM); - this.TM_NON_CA_SURG_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_SUM_DESC); - this.TM_NON_CA_SURG_MSK = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_MSK); - this.TM_NON_CA_SURG_YEAR = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_YEAR); - this.AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS); - this.TM_CA_SURG_98 = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_98); - this.TM_CA_SURG_98_MSK = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_98_MSK); - this.TM_CA_SURG_03 = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_03); - this.TM_CA_SURG_03_MSK = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_03_MSK); - this.TM_OTH_SURG_98 = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_98); - this.TM_OTH_SURG_98_MSK = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_98_MSK); - this.TM_OTH_SURG_03 = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_03); - this.TM_OTH_SURG_03_MSK = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_03_MSK); - this.TM_OTH_SURG_CD = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_CD); - this.TM_OTH_SURG_CD_DESC = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_CD_DESC); - this.TM_RGN_SCOP_98 = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_98); - this.TM_RGN_SCOP_98_MSK = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_98_MSK); - this.TM_RGN_SCOP_03 = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_03); - this.TM_RGN_SCOP_03_MSK = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_03_MSK); - this.TM_REGNODE_SCOP_CD = ClinicalValueUtil.defaultWithNA(TM_REGNODE_SCOP_CD); - this.TM_REGNODE_SCOP_DESC = ClinicalValueUtil.defaultWithNA(TM_REGNODE_SCOP_DESC); - this.TM_RECON_SURG = ClinicalValueUtil.defaultWithNA(TM_RECON_SURG); - this.TM_RECON_SURG_DESC = ClinicalValueUtil.defaultWithNA(TM_RECON_SURG_DESC); - this.TM_CA_SURG_YEAR = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_YEAR); - this.AGE_AT_TM_CA_SURG_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_CA_SURG_DATE_IN_DAYS); - this.TM_SURG_DEF_YEAR = ClinicalValueUtil.defaultWithNA(TM_SURG_DEF_YEAR); - this.AGE_AT_TM_SURG_DEF_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_SURG_DEF_DATE_IN_DAYS); - this.TM_REASON_NO_SURG = ClinicalValueUtil.defaultWithNA(TM_REASON_NO_SURG); - this.REASON_NO_SURG_DESC = ClinicalValueUtil.defaultWithNA(REASON_NO_SURG_DESC); - this.TM_PRIM_SURGEON = ClinicalValueUtil.defaultWithNA(TM_PRIM_SURGEON); - this.TM_PRIM_SURGEON_NAME = ClinicalValueUtil.defaultWithNA(TM_PRIM_SURGEON_NAME); - this.TM_ATN_DR_NO = ClinicalValueUtil.defaultWithNA(TM_ATN_DR_NO); - this.TM_ATN_DR_NAME = ClinicalValueUtil.defaultWithNA(TM_ATN_DR_NAME); - this.TM_REASON_NO_RAD = ClinicalValueUtil.defaultWithNA(TM_REASON_NO_RAD); - this.TM_REASON_NO_RAD_DESC = ClinicalValueUtil.defaultWithNA(TM_REASON_NO_RAD_DESC); - this.TM_RAD_STRT_YEAR = ClinicalValueUtil.defaultWithNA(TM_RAD_STRT_YEAR); - this.AGE_AT_TM_RAD_STRT_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_RAD_STRT_DATE_IN_DAYS); - this.TM_RAD_END_YEAR = ClinicalValueUtil.defaultWithNA(TM_RAD_END_YEAR); - this.AGE_AT_TM_RAD_END_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_RAD_END_DATE_IN_DAYS); - this.TM_RAD_TX_MOD = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_MOD); - this.TM_RAD_TX_MOD_DESC = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_MOD_DESC); - this.TM_BOOST_RAD_MOD = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_MOD); - this.TM_BOOST_RAD_MOD_DESC = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_MOD_DESC); - this.TM_LOC_RAD_TX = ClinicalValueUtil.defaultWithNA(TM_LOC_RAD_TX); - this.TM_LOC_RAD_TX_DESC = ClinicalValueUtil.defaultWithNA(TM_LOC_RAD_TX_DESC); - this.TM_RAD_TX_VOL = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_VOL); - this.TM_RAD_TX_VOL_DESC = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_VOL_DESC); - this.TM_NUM_TX_THIS_VOL = ClinicalValueUtil.defaultWithNA(TM_NUM_TX_THIS_VOL); - this.TM_NUM_TX_THIS_VOL_DESC = ClinicalValueUtil.defaultWithNA(TM_NUM_TX_THIS_VOL_DESC); - this.TM_REG_RAD_DOSE = ClinicalValueUtil.defaultWithNA(TM_REG_RAD_DOSE); - this.TM_REG_RAD_DOSE_DESC = ClinicalValueUtil.defaultWithNA(TM_REG_RAD_DOSE_DESC); - this.TM_BOOST_RAD_DOSE = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_DOSE); - this.TM_BOOST_RAD_DOSE_DESC = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_DOSE_DESC); - this.TM_RAD_SURG_SEQ = ClinicalValueUtil.defaultWithNA(TM_RAD_SURG_SEQ); - this.TM_RAD_SURG_SEQ_DESC = ClinicalValueUtil.defaultWithNA(TM_RAD_SURG_SEQ_DESC); - this.TM_RAD_MD = ClinicalValueUtil.defaultWithNA(TM_RAD_MD); - this.TM_RAD_MD_NAME = ClinicalValueUtil.defaultWithNA(TM_RAD_MD_NAME); - this.TM_SYST_STRT_YEAR = ClinicalValueUtil.defaultWithNA(TM_SYST_STRT_YEAR); - this.AGE_AT_TM_SYST_STRT_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_SYST_STRT_DATE_IN_DAYS); - this.TM_OTH_STRT_YEAR = ClinicalValueUtil.defaultWithNA(TM_OTH_STRT_YEAR); - this.AGE_AT_TM_OTH_STRT_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_OTH_STRT_DATE_IN_DAYS); - this.TM_CHEM_SUM = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM); - this.TM_CHEM_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM_DESC); - this.TM_CHEM_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM_MSK); - this.TM_CHEM_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM_MSK_DESC); - this.TM_TUMOR_SEQ = ClinicalValueUtil.defaultWithNA(TM_TUMOR_SEQ); - this.TM_HORM_SUM = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM); - this.TM_HORM_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM_DESC); - this.TM_HORM_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM_MSK); - this.TM_HORM_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM_MSK_DESC); - this.TM_BRM_SUM = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM); - this.TM_BRM_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM_DESC); - this.TM_BRM_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM_MSK); - this.TM_BRM_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM_MSK_DESC); - this.TM_OTH_SUM = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM); - this.TM_OTH_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM_DESC); - this.TM_OTH_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM_MSK); - this.TM_OTH_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM_MSK_DESC); - this.TM_PALLIA_PROC = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC); - this.TM_PALLIA_PROC_DESC = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC_DESC); - this.TM_PALLIA_PROC_MSK = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC_MSK); - this.TM_PALLIA_PROC_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC_MSK_DESC); - this.TM_ONCOLOGY_MD = ClinicalValueUtil.defaultWithNA(TM_ONCOLOGY_MD); - this.TM_ONCOLOGY_MD_NAME = ClinicalValueUtil.defaultWithNA(TM_ONCOLOGY_MD_NAME); - this.TM_PRCS_YEAR = ClinicalValueUtil.defaultWithNA(TM_PRCS_YEAR); - this.AGE_AT_TM_PRCS_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_PRCS_DATE_IN_DAYS); - this.TM_PATH_TEXT = ClinicalValueUtil.defaultWithNA(TM_PATH_TEXT); - this.TM_SURG_TEXT = ClinicalValueUtil.defaultWithNA(TM_SURG_TEXT); - this.TM_OVERRIDE_COM = ClinicalValueUtil.defaultWithNA(TM_OVERRIDE_COM); - this.TM_CSSIZE = ClinicalValueUtil.defaultWithNA(TM_CSSIZE); - this.TM_CSEXT = ClinicalValueUtil.defaultWithNA(TM_CSEXT); - this.TM_CSEXTEV = ClinicalValueUtil.defaultWithNA(TM_CSEXTEV); - this.TM_CSLMND = ClinicalValueUtil.defaultWithNA(TM_CSLMND); - this.TM_CSRGNEV = ClinicalValueUtil.defaultWithNA(TM_CSRGNEV); - this.TM_CSMETDX = ClinicalValueUtil.defaultWithNA(TM_CSMETDX); - this.TM_CSMETEV = ClinicalValueUtil.defaultWithNA(TM_CSMETEV); - this.TM_TSTAGE = ClinicalValueUtil.defaultWithNA(TM_TSTAGE); - this.TM_TSTAGE_DESC = ClinicalValueUtil.defaultWithNA(TM_TSTAGE_DESC); - this.TM_NSTAGE = ClinicalValueUtil.defaultWithNA(TM_NSTAGE); - this.TM_NSTAGE_DESC = ClinicalValueUtil.defaultWithNA(TM_NSTAGE_DESC); - this.TM_MSTAGE = ClinicalValueUtil.defaultWithNA(TM_MSTAGE); - this.TM_MSTAGE_DESC = ClinicalValueUtil.defaultWithNA(TM_MSTAGE_DESC); - this.TM_TBASIS = ClinicalValueUtil.defaultWithNA(TM_TBASIS); - this.TM_TBASIS_DESC = ClinicalValueUtil.defaultWithNA(TM_TBASIS_DESC); - this.TM_NBASIS = ClinicalValueUtil.defaultWithNA(TM_NBASIS); - this.TM_NBASIS_DESC = ClinicalValueUtil.defaultWithNA(TM_NBASIS_DESC); - this.TM_MBASIS = ClinicalValueUtil.defaultWithNA(TM_MBASIS); - this.TM_MBASIS_DESC = ClinicalValueUtil.defaultWithNA(TM_MBASIS_DESC); - this.TM_AJCC = ClinicalValueUtil.defaultWithNA(TM_AJCC); - this.TM_AJCC_DESC = ClinicalValueUtil.defaultWithNA(TM_AJCC_DESC); - this.TM_MSK_STG = ClinicalValueUtil.defaultWithNA(TM_MSK_STG); - } - - public String getPT_ID_ICDO(){ - return PT_ID_ICDO; - } - - public void setPT_ID_ICDO(String PT_ID_ICDO) { - this.PT_ID_ICDO = ClinicalValueUtil.defaultWithNA(PT_ID_ICDO); - } - - public Integer getTUMOR_YEAR(){ - return TUMOR_YEAR; - } - - public void setTUMOR_YEAR(Integer TUMOR_YEAR) { - this.TUMOR_YEAR = TUMOR_YEAR != null ? TUMOR_YEAR : -1; - } - - public String getDMP_ID_ICDO() { - return DMP_ID_ICDO; - } - - public void setDMP_ID_ICDO(String DMP_ID_ICDO) { - this.DMP_ID_ICDO = ClinicalValueUtil.defaultWithNA(DMP_ID_ICDO); - } - - public String getTM_TUMOR_SEQ_DESC() { - return TM_TUMOR_SEQ_DESC; - } - - public void setTM_TUMOR_SEQ_DESC(String TM_TUMOR_SEQ_DESC) { - this.TM_TUMOR_SEQ_DESC = ClinicalValueUtil.defaultWithNA(TM_TUMOR_SEQ_DESC); - } - - public String getTM_ACC_YEAR() { - return TM_ACC_YEAR; - } - - public void setTM_ACC_YEAR(String TM_ACC_YEAR) { - this.TM_ACC_YEAR = ClinicalValueUtil.defaultWithNA(TM_ACC_YEAR); - } - - public String getTM_FIRST_MSK_YEAR() { - return TM_FIRST_MSK_YEAR; - } - - public void setTM_FIRST_MSK_YEAR(String TM_FIRST_MSK_YEAR) { - this.TM_FIRST_MSK_YEAR = ClinicalValueUtil.defaultWithNA(TM_FIRST_MSK_YEAR); - } - - public String getAGE_AT_TM_FIRST_MSK_DATE_IN_DAYS() { - return AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_FIRST_MSK_DATE_IN_DAYS(String AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS) { - this.AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS); - } - - public String getTM_CASE_STS() { - return TM_CASE_STS; - } - - public void setTM_CASE_STS(String TM_CASE_STS) { - this.TM_CASE_STS = ClinicalValueUtil.defaultWithNA(TM_CASE_STS); - } - - public String getTM_CASE_STS_DESC() { - return TM_CASE_STS_DESC; - } - - public void setTM_CASE_STS_DESC(String TM_CASE_STS_DESC) { - this.TM_CASE_STS_DESC = ClinicalValueUtil.defaultWithNA(TM_CASE_STS_DESC); - } - - public String getTM_CASE_EFF_YEAR() { - return TM_CASE_EFF_YEAR; - } - - public void setTM_CASE_EFF_YEAR(String TM_CASE_EFF_YEAR) { - this.TM_CASE_EFF_YEAR = ClinicalValueUtil.defaultWithNA(TM_CASE_EFF_YEAR); - } - - public String getAGE_AT_TM_CASE_EFF_DATE_IN_DAYS() { - return AGE_AT_TM_CASE_EFF_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_CASE_EFF_DATE_IN_DAYS(String AGE_AT_TM_CASE_EFF_DATE_IN_DAYS) { - this.AGE_AT_TM_CASE_EFF_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_CASE_EFF_DATE_IN_DAYS); - } - - public String getTM_STATE_AT_DX() { - return TM_STATE_AT_DX; - } - - public void setTM_STATE_AT_DX(String TM_STATE_AT_DX) { - this.TM_STATE_AT_DX = ClinicalValueUtil.defaultWithNA(TM_STATE_AT_DX); - } - - public String getTM_SMOKING_HX() { - return TM_SMOKING_HX; - } - - public void setTM_SMOKING_HX(String TM_SMOKING_HX) { - this.TM_SMOKING_HX = ClinicalValueUtil.defaultWithNA(TM_SMOKING_HX); - } - - public String getTM_SMOKING_HX_DESC() { - return TM_SMOKING_HX_DESC; - } - - public void setTM_SMOKING_HX_DESC(String TM_SMOKING_HX_DESC) { - this.TM_SMOKING_HX_DESC = ClinicalValueUtil.defaultWithNA(TM_SMOKING_HX_DESC); - } - - public String getTM_OCCUPATION() { - return TM_OCCUPATION; - } - - public void setTM_OCCUPATION(String TM_OCCUPATION) { - this.TM_OCCUPATION = ClinicalValueUtil.defaultWithNA(TM_OCCUPATION); - } - - public String getTM_OCCUPATION_DESC() { - return TM_OCCUPATION_DESC; - } - - public void setTM_OCCUPATION_DESC(String TM_OCCUPATION_DESC) { - this.TM_OCCUPATION_DESC = ClinicalValueUtil.defaultWithNA(TM_OCCUPATION_DESC); - } - - public String getTM_FACILITY_FROM() { - return TM_FACILITY_FROM; - } - - public void setTM_FACILITY_FROM(String TM_FACILITY_FROM) { - this.TM_FACILITY_FROM = ClinicalValueUtil.defaultWithNA(TM_FACILITY_FROM); - } - - public String getFACILITY_FROM_DESC() { - return FACILITY_FROM_DESC; - } - - public void setFACILITY_FROM_DESC(String FACILITY_FROM_DESC) { - this.FACILITY_FROM_DESC = ClinicalValueUtil.defaultWithNA(FACILITY_FROM_DESC); - } - - public String getTM_FACILITY_TO() { - return TM_FACILITY_TO; - } - - public void setTM_FACILITY_TO(String TM_FACILITY_TO) { - this.TM_FACILITY_TO = ClinicalValueUtil.defaultWithNA(TM_FACILITY_TO); - } - - public String getFACILITY_TO_DESC() { - return FACILITY_TO_DESC; - } - - public void setFACILITY_TO_DESC(String FACILITY_TO_DESC) { - this.FACILITY_TO_DESC = ClinicalValueUtil.defaultWithNA(FACILITY_TO_DESC); - } - - public Integer getTM_DX_YEAR() { - return TM_DX_YEAR; - } - - public void setTM_DX_YEAR(Integer TM_DX_YEAR) { - this.TM_DX_YEAR = TM_DX_YEAR != null ? TM_DX_YEAR : 0; - } - - public Integer getAGE_AT_TM_DX_DATE_IN_DAYS() { - return AGE_AT_TM_DX_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_DX_DATE_IN_DAYS(Integer AGE_AT_TM_DX_DATE_IN_DAYS) { - this.AGE_AT_TM_DX_DATE_IN_DAYS = AGE_AT_TM_DX_DATE_IN_DAYS != null ? AGE_AT_TM_DX_DATE_IN_DAYS : -1; - } - - public String getTM_SITE_CD() { - return TM_SITE_CD; - } - - public void setTM_SITE_CD(String TM_SITE_CD) { - this.TM_SITE_CD = ClinicalValueUtil.defaultWithNA(TM_SITE_CD); - } - - public String getTM_SITE_DESC() { - return TM_SITE_DESC; - } - - public void setTM_SITE_DESC(String TM_SITE_DESC) { - this.TM_SITE_DESC = ClinicalValueUtil.defaultWithNA(TM_SITE_DESC); - } - - public String getTM_LATERALITY_CD() { - return TM_LATERALITY_CD; - } - - public void setTM_LATERALITY_CD(String TM_LATERALITY_CD) { - this.TM_LATERALITY_CD = ClinicalValueUtil.defaultWithNA(TM_LATERALITY_CD); - } - - public String getTM_LATERALITY_DESC() { - return TM_LATERALITY_DESC; - } - - public void setTM_LATERALITY_DESC(String TM_LATERALITY_DESC) { - this.TM_LATERALITY_DESC = ClinicalValueUtil.defaultWithNA(TM_LATERALITY_DESC); - } - - public String getTM_HIST_CD() { - return TM_HIST_CD; - } - - public void setTM_HIST_CD(String TM_HIST_CD) { - this.TM_HIST_CD = ClinicalValueUtil.defaultWithNA(TM_HIST_CD); - } - - public String getTM_HIST_DESC() { - return TM_HIST_DESC; - } - - public void setTM_HIST_DESC(String TM_HIST_DESC) { - this.TM_HIST_DESC = ClinicalValueUtil.defaultWithNA(TM_HIST_DESC); - } - - public String getTM_DX_CONFRM_CD() { - return TM_DX_CONFRM_CD; - } - - public void setTM_DX_CONFRM_CD(String TM_DX_CONFRM_CD) { - this.TM_DX_CONFRM_CD = ClinicalValueUtil.defaultWithNA(TM_DX_CONFRM_CD); - } - - public String getTM_DX_CONFRM_DESC() { - return TM_DX_CONFRM_DESC; - } - - public void setTM_DX_CONFRM_DESC(String TM_DX_CONFRM_DESC) { - this.TM_DX_CONFRM_DESC = ClinicalValueUtil.defaultWithNA(TM_DX_CONFRM_DESC); - } - - public String getTM_REGNODE_EXM_NO() { - return TM_REGNODE_EXM_NO; - } - - public void setTM_REGNODE_EXM_NO(String TM_REGNODE_EXM_NO) { - this.TM_REGNODE_EXM_NO = ClinicalValueUtil.defaultWithNA(TM_REGNODE_EXM_NO); - } - - public String getTM_REGNODE_POS_NO() { - return TM_REGNODE_POS_NO; - } - - public void setTM_REGNODE_POS_NO(String TM_REGNODE_POS_NO) { - this.TM_REGNODE_POS_NO = ClinicalValueUtil.defaultWithNA(TM_REGNODE_POS_NO); - } - - public String getTM_TUMOR_SIZE() { - return TM_TUMOR_SIZE; - } - - public void setTM_TUMOR_SIZE(String TM_TUMOR_SIZE) { - this.TM_TUMOR_SIZE = ClinicalValueUtil.defaultWithNA(TM_TUMOR_SIZE); - } - - public String getTM_RESID_TUMOR_CD() { - return TM_RESID_TUMOR_CD; - } - - public void setTM_RESID_TUMOR_CD(String TM_RESID_TUMOR_CD) { - this.TM_RESID_TUMOR_CD = ClinicalValueUtil.defaultWithNA(TM_RESID_TUMOR_CD); - } - - public String getTM_RESID_TUMOR_DESC() { - return TM_RESID_TUMOR_DESC; - } - - public void setTM_RESID_TUMOR_DESC(String TM_RESID_TUMOR_DESC) { - this.TM_RESID_TUMOR_DESC = ClinicalValueUtil.defaultWithNA(TM_RESID_TUMOR_DESC); - } - - public String getTM_GENERAL_STG() { - return TM_GENERAL_STG; - } - - public void setTM_GENERAL_STG(String TM_GENERAL_STG) { - this.TM_GENERAL_STG = ClinicalValueUtil.defaultWithNA(TM_GENERAL_STG); - } - - public String getTM_GENERAL_STG_DESC() { - return TM_GENERAL_STG_DESC; - } - - public void setTM_GENERAL_STG_DESC(String TM_GENERAL_STG_DESC) { - this.TM_GENERAL_STG_DESC = ClinicalValueUtil.defaultWithNA(TM_GENERAL_STG_DESC); - } - - public String getTM_TNM_EDITION() { - return TM_TNM_EDITION; - } - - public void setTM_TNM_EDITION(String TM_TNM_EDITION) { - this.TM_TNM_EDITION = ClinicalValueUtil.defaultWithNA(TM_TNM_EDITION); - } - - public String getTM_TNM_EDITION_DESC() { - return TM_TNM_EDITION_DESC; - } - - public void setTM_TNM_EDITION_DESC(String TM_TNM_EDITION_DESC) { - this.TM_TNM_EDITION_DESC = ClinicalValueUtil.defaultWithNA(TM_TNM_EDITION_DESC); - } - - public String getTM_CLIN_TNM_T() { - return TM_CLIN_TNM_T; - } - - public void setTM_CLIN_TNM_T(String TM_CLIN_TNM_T) { - this.TM_CLIN_TNM_T = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_T); - } - - public String getTM_CLIN_TNM_T_DESC() { - return TM_CLIN_TNM_T_DESC; - } - - public void setTM_CLIN_TNM_T_DESC(String TM_CLIN_TNM_T_DESC) { - this.TM_CLIN_TNM_T_DESC = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_T_DESC); - } - - public String getTM_CLIN_TNM_N() { - return TM_CLIN_TNM_N; - } - - public void setTM_CLIN_TNM_N(String TM_CLIN_TNM_N) { - this.TM_CLIN_TNM_N = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_N); - } - - public String getTM_CLIN_TNM_N_DESC() { - return TM_CLIN_TNM_N_DESC; - } - - public void setTM_CLIN_TNM_N_DESC(String TM_CLIN_TNM_N_DESC) { - this.TM_CLIN_TNM_N_DESC = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_N_DESC); - } - - public String getTM_CLIN_TNM_M() { - return TM_CLIN_TNM_M; - } - - public void setTM_CLIN_TNM_M(String TM_CLIN_TNM_M) { - this.TM_CLIN_TNM_M = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_M); - } - - public String getTM_CLIN_TNM_M_DESC() { - return TM_CLIN_TNM_M_DESC; - } - - public void setTM_CLIN_TNM_M_DESC(String TM_CLIN_TNM_M_DESC) { - this.TM_CLIN_TNM_M_DESC = ClinicalValueUtil.defaultWithNA(TM_CLIN_TNM_M_DESC); - } - - public String getTM_CLIN_STG_GRP() { - return TM_CLIN_STG_GRP; - } - - public void setTM_CLIN_STG_GRP(String TM_CLIN_STG_GRP) { - this.TM_CLIN_STG_GRP = ClinicalValueUtil.defaultWithNA(TM_CLIN_STG_GRP); - } - - public String getTM_PATH_TNM_T() { - return TM_PATH_TNM_T; - } - - public void setTM_PATH_TNM_T(String TM_PATH_TNM_T) { - this.TM_PATH_TNM_T = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_T); - } - - public String getTM_PATH_TNM_T_DESC() { - return TM_PATH_TNM_T_DESC; - } - - public void setTM_PATH_TNM_T_DESC(String TM_PATH_TNM_T_DESC) { - this.TM_PATH_TNM_T_DESC = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_T_DESC); - } - - public String getTM_PATH_TNM_N() { - return TM_PATH_TNM_N; - } - - public void setTM_PATH_TNM_N(String TM_PATH_TNM_N) { - this.TM_PATH_TNM_N = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_N); - } - - public String getTM_PATH_TNM_N_DESC() { - return TM_PATH_TNM_N_DESC; - } - - public void setTM_PATH_TNM_N_DESC(String TM_PATH_TNM_N_DESC) { - this.TM_PATH_TNM_N_DESC = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_N_DESC); - } - - public String getTM_PATH_TNM_M() { - return TM_PATH_TNM_M; - } - - public void setTM_PATH_TNM_M(String TM_PATH_TNM_M) { - this.TM_PATH_TNM_M = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_M); - } - - public String getTM_PATH_TNM_M_DESC() { - return TM_PATH_TNM_M_DESC; - } - - public void setTM_PATH_TNM_M_DESC(String TM_PATH_TNM_M_DESC) { - this.TM_PATH_TNM_M_DESC = ClinicalValueUtil.defaultWithNA(TM_PATH_TNM_M_DESC); - } - - public String getTM_PATH_STG_GRP() { - return TM_PATH_STG_GRP; - } - - public void setTM_PATH_STG_GRP(String TM_PATH_STG_GRP) { - this.TM_PATH_STG_GRP = ClinicalValueUtil.defaultWithNA(TM_PATH_STG_GRP); - } - - public String getTM_PATH_RPT_AV() { - return TM_PATH_RPT_AV; - } - - public void setTM_PATH_RPT_AV(String TM_PATH_RPT_AV) { - this.TM_PATH_RPT_AV = ClinicalValueUtil.defaultWithNA(TM_PATH_RPT_AV); - } - - public String getTM_CA_STS_AT_ACC() { - return TM_CA_STS_AT_ACC; - } - - public void setTM_CA_STS_AT_ACC(String TM_CA_STS_AT_ACC) { - this.TM_CA_STS_AT_ACC = ClinicalValueUtil.defaultWithNA(TM_CA_STS_AT_ACC); - } - - public String getTM_FIRST_RECUR_YEAR() { - return TM_FIRST_RECUR_YEAR; - } - - public void setTM_FIRST_RECUR_YEAR(String TM_FIRST_RECUR_YEAR) { - this.TM_FIRST_RECUR_YEAR = ClinicalValueUtil.defaultWithNA(TM_FIRST_RECUR_YEAR); - } - - public String getAGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS() { - return AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS(String AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS) { - this.AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS); - } - - public String getTM_FIRST_RECUR_TYP() { - return TM_FIRST_RECUR_TYP; - } - - public void setTM_FIRST_RECUR_TYP(String TM_FIRST_RECUR_TYP) { - this.TM_FIRST_RECUR_TYP = ClinicalValueUtil.defaultWithNA(TM_FIRST_RECUR_TYP); - } - - public String getTM_ADM_YEAR() { - return TM_ADM_YEAR; - } - - public void setTM_ADM_YEAR(String TM_ADM_YEAR) { - this.TM_ADM_YEAR = ClinicalValueUtil.defaultWithNA(TM_ADM_YEAR); - } - - public String getAGE_AT_TM_ADM_DATE_IN_DAYS() { - return AGE_AT_TM_ADM_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_ADM_DATE_IN_DAYS(String AGE_AT_TM_ADM_DATE_IN_DAYS) { - this.AGE_AT_TM_ADM_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_ADM_DATE_IN_DAYS); - } - - public String getTM_DSCH_YEAR() { - return TM_DSCH_YEAR; - } - - public void setTM_DSCH_YEAR(String TM_DSCH_YEAR) { - this.TM_DSCH_YEAR = ClinicalValueUtil.defaultWithNA(TM_DSCH_YEAR); - } - - public String getAGE_AT_TM_DSCH_DATE_IN_DAYS() { - return AGE_AT_TM_DSCH_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_DSCH_DATE_IN_DAYS(String AGE_AT_TM_DSCH_DATE_IN_DAYS) { - this.AGE_AT_TM_DSCH_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_DSCH_DATE_IN_DAYS); - } - - public String getTM_SURG_DSCH_YEAR() { - return TM_SURG_DSCH_YEAR; - } - - public void setTM_SURG_DSCH_YEAR(String TM_SURG_DSCH_YEAR) { - this.TM_SURG_DSCH_YEAR = ClinicalValueUtil.defaultWithNA(TM_SURG_DSCH_YEAR); - } - - public String getAGE_AT_TM_SURG_DSCH_DATE_IN_DAYS() { - return AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_SURG_DSCH_DATE_IN_DAYS(String AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS) { - this.AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS); - } - - public String getTM_READM_WTHN_30D() { - return TM_READM_WTHN_30D; - } - - public void setTM_READM_WTHN_30D(String TM_READM_WTHN_30D) { - this.TM_READM_WTHN_30D = ClinicalValueUtil.defaultWithNA(TM_READM_WTHN_30D); - } - - public String getTM_FIRST_TX_YEAR() { - return TM_FIRST_TX_YEAR; - } - - public void setTM_FIRST_TX_YEAR(String TM_FIRST_TX_YEAR) { - this.TM_FIRST_TX_YEAR = ClinicalValueUtil.defaultWithNA(TM_FIRST_TX_YEAR); - } - - public String getAGE_AT_TM_FIRST_TX_DATE_IN_DAYS() { - return AGE_AT_TM_FIRST_TX_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_FIRST_TX_DATE_IN_DAYS(String AGE_AT_TM_FIRST_TX_DATE_IN_DAYS) { - this.AGE_AT_TM_FIRST_TX_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_FIRST_TX_DATE_IN_DAYS); - } - - public String getTM_NON_CA_SURG_SUM() { - return TM_NON_CA_SURG_SUM; - } - - public void setTM_NON_CA_SURG_SUM(String TM_NON_CA_SURG_SUM) { - this.TM_NON_CA_SURG_SUM = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_SUM); - } - - public String getTM_NON_CA_SURG_SUM_DESC() { - return TM_NON_CA_SURG_SUM_DESC; - } - - public void setTM_NON_CA_SURG_SUM_DESC(String TM_NON_CA_SURG_SUM_DESC) { - this.TM_NON_CA_SURG_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_SUM_DESC); - } - - public String getTM_NON_CA_SURG_MSK() { - return TM_NON_CA_SURG_MSK; - } - - public void setTM_NON_CA_SURG_MSK(String TM_NON_CA_SURG_MSK) { - this.TM_NON_CA_SURG_MSK = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_MSK); - } - - public String getTM_NON_CA_SURG_YEAR() { - return TM_NON_CA_SURG_YEAR; - } - - public void setTM_NON_CA_SURG_YEAR(String TM_NON_CA_SURG_YEAR) { - this.TM_NON_CA_SURG_YEAR = ClinicalValueUtil.defaultWithNA(TM_NON_CA_SURG_YEAR); - } - - public String getAGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS() { - return AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS(String AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS) { - this.AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS); - } - - public String getTM_CA_SURG_98() { - return TM_CA_SURG_98; - } - - public void setTM_CA_SURG_98(String TM_CA_SURG_98) { - this.TM_CA_SURG_98 = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_98); - } - - public String getTM_CA_SURG_98_MSK() { - return TM_CA_SURG_98_MSK; - } - - public void setTM_CA_SURG_98_MSK(String TM_CA_SURG_98_MSK) { - this.TM_CA_SURG_98_MSK = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_98_MSK); - } - - public String getTM_CA_SURG_03() { - return TM_CA_SURG_03; - } - - public void setTM_CA_SURG_03(String TM_CA_SURG_03) { - this.TM_CA_SURG_03 = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_03); - } - - public String getTM_CA_SURG_03_MSK() { - return TM_CA_SURG_03_MSK; - } - - public void setTM_CA_SURG_03_MSK(String TM_CA_SURG_03_MSK) { - this.TM_CA_SURG_03_MSK = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_03_MSK); - } - - public String getTM_OTH_SURG_98() { - return TM_OTH_SURG_98; - } - - public void setTM_OTH_SURG_98(String TM_OTH_SURG_98) { - this.TM_OTH_SURG_98 = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_98); - } - - public String getTM_OTH_SURG_98_MSK() { - return TM_OTH_SURG_98_MSK; - } - - public void setTM_OTH_SURG_98_MSK(String TM_OTH_SURG_98_MSK) { - this.TM_OTH_SURG_98_MSK = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_98_MSK); - } - - public String getTM_OTH_SURG_03() { - return TM_OTH_SURG_03; - } - - public void setTM_OTH_SURG_03(String TM_OTH_SURG_03) { - this.TM_OTH_SURG_03 = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_03); - } - - public String getTM_OTH_SURG_03_MSK() { - return TM_OTH_SURG_03_MSK; - } - - public void setTM_OTH_SURG_03_MSK(String TM_OTH_SURG_03_MSK) { - this.TM_OTH_SURG_03_MSK = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_03_MSK); - } - - public String getTM_OTH_SURG_CD() { - return TM_OTH_SURG_CD; - } - - public void setTM_OTH_SURG_CD(String TM_OTH_SURG_CD) { - this.TM_OTH_SURG_CD = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_CD); - } - - public String getTM_OTH_SURG_CD_DESC() { - return TM_OTH_SURG_CD_DESC; - } - - public void setTM_OTH_SURG_CD_DESC(String TM_OTH_SURG_CD_DESC) { - this.TM_OTH_SURG_CD_DESC = ClinicalValueUtil.defaultWithNA(TM_OTH_SURG_CD_DESC); - } - - public String getTM_RGN_SCOP_98() { - return TM_RGN_SCOP_98; - } - - public void setTM_RGN_SCOP_98(String TM_RGN_SCOP_98) { - this.TM_RGN_SCOP_98 = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_98); - } - - public String getTM_RGN_SCOP_98_MSK() { - return TM_RGN_SCOP_98_MSK; - } - - public void setTM_RGN_SCOP_98_MSK(String TM_RGN_SCOP_98_MSK) { - this.TM_RGN_SCOP_98_MSK = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_98_MSK); - } - - public String getTM_RGN_SCOP_03() { - return TM_RGN_SCOP_03; - } - - public void setTM_RGN_SCOP_03(String TM_RGN_SCOP_03) { - this.TM_RGN_SCOP_03 = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_03); - } - - public String getTM_RGN_SCOP_03_MSK() { - return TM_RGN_SCOP_03_MSK; - } - - public void setTM_RGN_SCOP_03_MSK(String TM_RGN_SCOP_03_MSK) { - this.TM_RGN_SCOP_03_MSK = ClinicalValueUtil.defaultWithNA(TM_RGN_SCOP_03_MSK); - } - - public String getTM_REGNODE_SCOP_CD() { - return TM_REGNODE_SCOP_CD; - } - - public void setTM_REGNODE_SCOP_CD(String TM_REGNODE_SCOP_CD) { - this.TM_REGNODE_SCOP_CD = ClinicalValueUtil.defaultWithNA(TM_REGNODE_SCOP_CD); - } - - public String getTM_REGNODE_SCOP_DESC() { - return TM_REGNODE_SCOP_DESC; - } - - public void setTM_REGNODE_SCOP_DESC(String TM_REGNODE_SCOP_DESC) { - this.TM_REGNODE_SCOP_DESC = ClinicalValueUtil.defaultWithNA(TM_REGNODE_SCOP_DESC); - } - - public String getTM_RECON_SURG() { - return TM_RECON_SURG; - } - - public void setTM_RECON_SURG(String TM_RECON_SURG) { - this.TM_RECON_SURG = ClinicalValueUtil.defaultWithNA(TM_RECON_SURG); - } - - public String getTM_RECON_SURG_DESC() { - return TM_RECON_SURG_DESC; - } - - public void setTM_RECON_SURG_DESC(String TM_RECON_SURG_DESC) { - this.TM_RECON_SURG_DESC = ClinicalValueUtil.defaultWithNA(TM_RECON_SURG_DESC); - } - - public String getTM_CA_SURG_YEAR() { - return TM_CA_SURG_YEAR; - } - - public void setTM_CA_SURG_YEAR(String TM_CA_SURG_YEAR) { - this.TM_CA_SURG_YEAR = ClinicalValueUtil.defaultWithNA(TM_CA_SURG_YEAR); - } - - public String getAGE_AT_TM_CA_SURG_DATE_IN_DAYS() { - return AGE_AT_TM_CA_SURG_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_CA_SURG_DATE_IN_DAYS(String AGE_AT_TM_CA_SURG_DATE_IN_DAYS) { - this.AGE_AT_TM_CA_SURG_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_CA_SURG_DATE_IN_DAYS); - } - - public String getTM_SURG_DEF_YEAR() { - return TM_SURG_DEF_YEAR; - } - - public void setTM_SURG_DEF_YEAR(String TM_SURG_DEF_YEAR) { - this.TM_SURG_DEF_YEAR = ClinicalValueUtil.defaultWithNA(TM_SURG_DEF_YEAR); - } - - public String getAGE_AT_TM_SURG_DEF_DATE_IN_DAYS() { - return AGE_AT_TM_SURG_DEF_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_SURG_DEF_DATE_IN_DAYS(String AGE_AT_TM_SURG_DEF_DATE_IN_DAYS) { - this.AGE_AT_TM_SURG_DEF_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_SURG_DEF_DATE_IN_DAYS); - } - - public String getTM_REASON_NO_SURG() { - return TM_REASON_NO_SURG; - } - - public void setTM_REASON_NO_SURG(String TM_REASON_NO_SURG) { - this.TM_REASON_NO_SURG = ClinicalValueUtil.defaultWithNA(TM_REASON_NO_SURG); - } - - public String getREASON_NO_SURG_DESC() { - return REASON_NO_SURG_DESC; - } - - public void setREASON_NO_SURG_DESC(String REASON_NO_SURG_DESC) { - this.REASON_NO_SURG_DESC = ClinicalValueUtil.defaultWithNA(REASON_NO_SURG_DESC); - } - - public String getTM_PRIM_SURGEON() { - return TM_PRIM_SURGEON; - } - - public void setTM_PRIM_SURGEON(String TM_PRIM_SURGEON) { - this.TM_PRIM_SURGEON = ClinicalValueUtil.defaultWithNA(TM_PRIM_SURGEON); - } - - public String getTM_PRIM_SURGEON_NAME() { - return TM_PRIM_SURGEON_NAME; - } - - public void setTM_PRIM_SURGEON_NAME(String TM_PRIM_SURGEON_NAME) { - this.TM_PRIM_SURGEON_NAME = ClinicalValueUtil.defaultWithNA(TM_PRIM_SURGEON_NAME); - } - - public String getTM_ATN_DR_NO() { - return TM_ATN_DR_NO; - } - - public void setTM_ATN_DR_NO(String TM_ATN_DR_NO) { - this.TM_ATN_DR_NO = ClinicalValueUtil.defaultWithNA(TM_ATN_DR_NO); - } - - public String getTM_ATN_DR_NAME() { - return TM_ATN_DR_NAME; - } - - public void setTM_ATN_DR_NAME(String TM_ATN_DR_NAME) { - this.TM_ATN_DR_NAME = ClinicalValueUtil.defaultWithNA(TM_ATN_DR_NAME); - } - - public String getTM_REASON_NO_RAD() { - return TM_REASON_NO_RAD; - } - - public void setTM_REASON_NO_RAD(String TM_REASON_NO_RAD) { - this.TM_REASON_NO_RAD = ClinicalValueUtil.defaultWithNA(TM_REASON_NO_RAD); - } - - public String getTM_REASON_NO_RAD_DESC() { - return TM_REASON_NO_RAD_DESC; - } - - public void setTM_REASON_NO_RAD_DESC(String TM_REASON_NO_RAD_DESC) { - this.TM_REASON_NO_RAD_DESC = ClinicalValueUtil.defaultWithNA(TM_REASON_NO_RAD_DESC); - } - - public String getTM_RAD_STRT_YEAR() { - return TM_RAD_STRT_YEAR; - } - - public void setTM_RAD_STRT_YEAR(String TM_RAD_STRT_YEAR) { - this.TM_RAD_STRT_YEAR = ClinicalValueUtil.defaultWithNA(TM_RAD_STRT_YEAR); - } - - public String getAGE_AT_TM_RAD_STRT_DATE_IN_DAYS() { - return AGE_AT_TM_RAD_STRT_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_RAD_STRT_DATE_IN_DAYS(String AGE_AT_TM_RAD_STRT_DATE_IN_DAYS) { - this.AGE_AT_TM_RAD_STRT_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_RAD_STRT_DATE_IN_DAYS); - } - - public String getTM_RAD_END_YEAR() { - return TM_RAD_END_YEAR; - } - - public void setTM_RAD_END_YEAR(String TM_RAD_END_YEAR) { - this.TM_RAD_END_YEAR = ClinicalValueUtil.defaultWithNA(TM_RAD_END_YEAR); - } - - public String getAGE_AT_TM_RAD_END_DATE_IN_DAYS() { - return AGE_AT_TM_RAD_END_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_RAD_END_DATE_IN_DAYS(String AGE_AT_TM_RAD_END_DATE_IN_DAYS) { - this.AGE_AT_TM_RAD_END_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_RAD_END_DATE_IN_DAYS); - } - - public String getTM_RAD_TX_MOD() { - return TM_RAD_TX_MOD; - } - - public void setTM_RAD_TX_MOD(String TM_RAD_TX_MOD) { - this.TM_RAD_TX_MOD = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_MOD); - } - - public String getTM_RAD_TX_MOD_DESC() { - return TM_RAD_TX_MOD_DESC; - } - - public void setTM_RAD_TX_MOD_DESC(String TM_RAD_TX_MOD_DESC) { - this.TM_RAD_TX_MOD_DESC = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_MOD_DESC); - } - - public String getTM_BOOST_RAD_MOD() { - return TM_BOOST_RAD_MOD; - } - - public void setTM_BOOST_RAD_MOD(String TM_BOOST_RAD_MOD) { - this.TM_BOOST_RAD_MOD = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_MOD); - } - - public String getTM_BOOST_RAD_MOD_DESC() { - return TM_BOOST_RAD_MOD_DESC; - } - - public void setTM_BOOST_RAD_MOD_DESC(String TM_BOOST_RAD_MOD_DESC) { - this.TM_BOOST_RAD_MOD_DESC = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_MOD_DESC); - } - - public String getTM_LOC_RAD_TX() { - return TM_LOC_RAD_TX; - } - - public void setTM_LOC_RAD_TX(String TM_LOC_RAD_TX) { - this.TM_LOC_RAD_TX = ClinicalValueUtil.defaultWithNA(TM_LOC_RAD_TX); - } - - public String getTM_LOC_RAD_TX_DESC() { - return TM_LOC_RAD_TX_DESC; - } - - public void setTM_LOC_RAD_TX_DESC(String TM_LOC_RAD_TX_DESC) { - this.TM_LOC_RAD_TX_DESC = ClinicalValueUtil.defaultWithNA(TM_LOC_RAD_TX_DESC); - } - - public String getTM_RAD_TX_VOL() { - return TM_RAD_TX_VOL; - } - - public void setTM_RAD_TX_VOL(String TM_RAD_TX_VOL) { - this.TM_RAD_TX_VOL = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_VOL); - } - - public String getTM_RAD_TX_VOL_DESC() { - return TM_RAD_TX_VOL_DESC; - } - - public void setTM_RAD_TX_VOL_DESC(String TM_RAD_TX_VOL_DESC) { - this.TM_RAD_TX_VOL_DESC = ClinicalValueUtil.defaultWithNA(TM_RAD_TX_VOL_DESC); - } - - public String getTM_NUM_TX_THIS_VOL() { - return TM_NUM_TX_THIS_VOL; - } - - public void setTM_NUM_TX_THIS_VOL(String TM_NUM_TX_THIS_VOL) { - this.TM_NUM_TX_THIS_VOL = ClinicalValueUtil.defaultWithNA(TM_NUM_TX_THIS_VOL); - } - - public String getTM_NUM_TX_THIS_VOL_DESC() { - return TM_NUM_TX_THIS_VOL_DESC; - } - - public void setTM_NUM_TX_THIS_VOL_DESC(String TM_NUM_TX_THIS_VOL_DESC) { - this.TM_NUM_TX_THIS_VOL_DESC = ClinicalValueUtil.defaultWithNA(TM_NUM_TX_THIS_VOL_DESC); - } - - public String getTM_REG_RAD_DOSE() { - return TM_REG_RAD_DOSE; - } - - public void setTM_REG_RAD_DOSE(String TM_REG_RAD_DOSE) { - this.TM_REG_RAD_DOSE = ClinicalValueUtil.defaultWithNA(TM_REG_RAD_DOSE); - } - - public String getTM_REG_RAD_DOSE_DESC() { - return TM_REG_RAD_DOSE_DESC; - } - - public void setTM_REG_RAD_DOSE_DESC(String TM_REG_RAD_DOSE_DESC) { - this.TM_REG_RAD_DOSE_DESC = ClinicalValueUtil.defaultWithNA(TM_REG_RAD_DOSE_DESC); - } - - public String getTM_BOOST_RAD_DOSE() { - return TM_BOOST_RAD_DOSE; - } - - public void setTM_BOOST_RAD_DOSE(String TM_BOOST_RAD_DOSE) { - this.TM_BOOST_RAD_DOSE = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_DOSE); - } - - public String getTM_BOOST_RAD_DOSE_DESC() { - return TM_BOOST_RAD_DOSE_DESC; - } - - public void setTM_BOOST_RAD_DOSE_DESC(String TM_BOOST_RAD_DOSE_DESC) { - this.TM_BOOST_RAD_DOSE_DESC = ClinicalValueUtil.defaultWithNA(TM_BOOST_RAD_DOSE_DESC); - } - - public String getTM_RAD_SURG_SEQ() { - return TM_RAD_SURG_SEQ; - } - - public void setTM_RAD_SURG_SEQ(String TM_RAD_SURG_SEQ) { - this.TM_RAD_SURG_SEQ = ClinicalValueUtil.defaultWithNA(TM_RAD_SURG_SEQ); - } - - public String getTM_RAD_SURG_SEQ_DESC() { - return TM_RAD_SURG_SEQ_DESC; - } - - public void setTM_RAD_SURG_SEQ_DESC(String TM_RAD_SURG_SEQ_DESC) { - this.TM_RAD_SURG_SEQ_DESC = ClinicalValueUtil.defaultWithNA(TM_RAD_SURG_SEQ_DESC); - } - - public String getTM_RAD_MD() { - return TM_RAD_MD; - } - - public void setTM_RAD_MD(String TM_RAD_MD) { - this.TM_RAD_MD = ClinicalValueUtil.defaultWithNA(TM_RAD_MD); - } - - public String getTM_RAD_MD_NAME() { - return TM_RAD_MD_NAME; - } - - public void setTM_RAD_MD_NAME(String TM_RAD_MD_NAME) { - this.TM_RAD_MD_NAME = ClinicalValueUtil.defaultWithNA(TM_RAD_MD_NAME); - } - - public String getTM_SYST_STRT_YEAR() { - return TM_SYST_STRT_YEAR; - } - - public void setTM_SYST_STRT_YEAR(String TM_SYST_STRT_YEAR) { - this.TM_SYST_STRT_YEAR = ClinicalValueUtil.defaultWithNA(TM_SYST_STRT_YEAR); - } - - public String getAGE_AT_TM_SYST_STRT_DATE_IN_DAYS() { - return AGE_AT_TM_SYST_STRT_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_SYST_STRT_DATE_IN_DAYS(String AGE_AT_TM_SYST_STRT_DATE_IN_DAYS) { - this.AGE_AT_TM_SYST_STRT_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_SYST_STRT_DATE_IN_DAYS); - } - - public String getTM_OTH_STRT_YEAR() { - return TM_OTH_STRT_YEAR; - } - - public void setTM_OTH_STRT_YEAR(String TM_OTH_STRT_YEAR) { - this.TM_OTH_STRT_YEAR = ClinicalValueUtil.defaultWithNA(TM_OTH_STRT_YEAR); - } - - public String getAGE_AT_TM_OTH_STRT_DATE_IN_DAYS() { - return AGE_AT_TM_OTH_STRT_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_OTH_STRT_DATE_IN_DAYS(String AGE_AT_TM_OTH_STRT_DATE_IN_DAYS) { - this.AGE_AT_TM_OTH_STRT_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_OTH_STRT_DATE_IN_DAYS); - } - - public String getTM_CHEM_SUM() { - return TM_CHEM_SUM; - } - - public void setTM_CHEM_SUM(String TM_CHEM_SUM) { - this.TM_CHEM_SUM = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM); - } - - public String getTM_CHEM_SUM_DESC() { - return TM_CHEM_SUM_DESC; - } - - public void setTM_CHEM_SUM_DESC(String TM_CHEM_SUM_DESC) { - this.TM_CHEM_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM_DESC); - } - - public String getTM_CHEM_SUM_MSK() { - return TM_CHEM_SUM_MSK; - } - - public void setTM_CHEM_SUM_MSK(String TM_CHEM_SUM_MSK) { - this.TM_CHEM_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM_MSK); - } - - public String getTM_CHEM_SUM_MSK_DESC() { - return TM_CHEM_SUM_MSK_DESC; - } - - public void setTM_CHEM_SUM_MSK_DESC(String TM_CHEM_SUM_MSK_DESC) { - this.TM_CHEM_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_CHEM_SUM_MSK_DESC); - } - - public String getTM_TUMOR_SEQ() { - return TM_TUMOR_SEQ; - } - - public void setTM_TUMOR_SEQ(String TM_TUMOR_SEQ) { - this.TM_TUMOR_SEQ = ClinicalValueUtil.defaultWithNA(TM_TUMOR_SEQ); - } - - public String getTM_HORM_SUM() { - return TM_HORM_SUM; - } - - public void setTM_HORM_SUM(String TM_HORM_SUM) { - this.TM_HORM_SUM = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM); - } - - public String getTM_HORM_SUM_DESC() { - return TM_HORM_SUM_DESC; - } - - public void setTM_HORM_SUM_DESC(String TM_HORM_SUM_DESC) { - this.TM_HORM_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM_DESC); - } - - public String getTM_HORM_SUM_MSK() { - return TM_HORM_SUM_MSK; - } - - public void setTM_HORM_SUM_MSK(String TM_HORM_SUM_MSK) { - this.TM_HORM_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM_MSK); - } - - public String getTM_HORM_SUM_MSK_DESC() { - return TM_HORM_SUM_MSK_DESC; - } - - public void setTM_HORM_SUM_MSK_DESC(String TM_HORM_SUM_MSK_DESC) { - this.TM_HORM_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_HORM_SUM_MSK_DESC); - } - - public String getTM_BRM_SUM() { - return TM_BRM_SUM; - } - - public void setTM_BRM_SUM(String TM_BRM_SUM) { - this.TM_BRM_SUM = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM); - } - - public String getTM_BRM_SUM_DESC() { - return TM_BRM_SUM_DESC; - } - - public void setTM_BRM_SUM_DESC(String TM_BRM_SUM_DESC) { - this.TM_BRM_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM_DESC); - } - - public String getTM_BRM_SUM_MSK() { - return TM_BRM_SUM_MSK; - } - - public void setTM_BRM_SUM_MSK(String TM_BRM_SUM_MSK) { - this.TM_BRM_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM_MSK); - } - - public String getTM_BRM_SUM_MSK_DESC() { - return TM_BRM_SUM_MSK_DESC; - } - - public void setTM_BRM_SUM_MSK_DESC(String TM_BRM_SUM_MSK_DESC) { - this.TM_BRM_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_BRM_SUM_MSK_DESC); - } - - public String getTM_OTH_SUM() { - return TM_OTH_SUM; - } - - public void setTM_OTH_SUM(String TM_OTH_SUM) { - this.TM_OTH_SUM = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM); - } - - public String getTM_OTH_SUM_DESC() { - return TM_OTH_SUM_DESC; - } - - public void setTM_OTH_SUM_DESC(String TM_OTH_SUM_DESC) { - this.TM_OTH_SUM_DESC = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM_DESC); - } - - public String getTM_OTH_SUM_MSK() { - return TM_OTH_SUM_MSK; - } - - public void setTM_OTH_SUM_MSK(String TM_OTH_SUM_MSK) { - this.TM_OTH_SUM_MSK = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM_MSK); - } - - public String getTM_OTH_SUM_MSK_DESC() { - return TM_OTH_SUM_MSK_DESC; - } - - public void setTM_OTH_SUM_MSK_DESC(String TM_OTH_SUM_MSK_DESC) { - this.TM_OTH_SUM_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_OTH_SUM_MSK_DESC); - } - - public String getTM_PALLIA_PROC() { - return TM_PALLIA_PROC; - } - - public void setTM_PALLIA_PROC(String TM_PALLIA_PROC) { - this.TM_PALLIA_PROC = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC); - } - - public String getTM_PALLIA_PROC_DESC() { - return TM_PALLIA_PROC_DESC; - } - - public void setTM_PALLIA_PROC_DESC(String TM_PALLIA_PROC_DESC) { - this.TM_PALLIA_PROC_DESC = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC_DESC); - } - - public String getTM_PALLIA_PROC_MSK() { - return TM_PALLIA_PROC_MSK; - } - - public void setTM_PALLIA_PROC_MSK(String TM_PALLIA_PROC_MSK) { - this.TM_PALLIA_PROC_MSK = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC_MSK); - } - - public String getTM_PALLIA_PROC_MSK_DESC() { - return TM_PALLIA_PROC_MSK_DESC; - } - - public void setTM_PALLIA_PROC_MSK_DESC(String TM_PALLIA_PROC_MSK_DESC) { - this.TM_PALLIA_PROC_MSK_DESC = ClinicalValueUtil.defaultWithNA(TM_PALLIA_PROC_MSK_DESC); - } - - public String getTM_ONCOLOGY_MD() { - return TM_ONCOLOGY_MD; - } - - public void setTM_ONCOLOGY_MD(String TM_ONCOLOGY_MD) { - this.TM_ONCOLOGY_MD = ClinicalValueUtil.defaultWithNA(TM_ONCOLOGY_MD); - } - - public String getTM_ONCOLOGY_MD_NAME() { - return TM_ONCOLOGY_MD_NAME; - } - - public void setTM_ONCOLOGY_MD_NAME(String TM_ONCOLOGY_MD_NAME) { - this.TM_ONCOLOGY_MD_NAME = ClinicalValueUtil.defaultWithNA(TM_ONCOLOGY_MD_NAME); - } - - public String getTM_PRCS_YEAR() { - return TM_PRCS_YEAR; - } - - public void setTM_PRCS_YEAR(String TM_PRCS_YEAR) { - this.TM_PRCS_YEAR = ClinicalValueUtil.defaultWithNA(TM_PRCS_YEAR); - } - - public String getAGE_AT_TM_PRCS_DATE_IN_DAYS() { - return AGE_AT_TM_PRCS_DATE_IN_DAYS; - } - - public void setAGE_AT_TM_PRCS_DATE_IN_DAYS(String AGE_AT_TM_PRCS_DATE_IN_DAYS) { - this.AGE_AT_TM_PRCS_DATE_IN_DAYS = ClinicalValueUtil.defaultWithNA(AGE_AT_TM_PRCS_DATE_IN_DAYS); - } - - public String getTM_PATH_TEXT() { - return TM_PATH_TEXT; - } - - public void setTM_PATH_TEXT(String TM_PATH_TEXT) { - this.TM_PATH_TEXT = ClinicalValueUtil.defaultWithNA(TM_PATH_TEXT); - } - - public String getTM_SURG_TEXT() { - return TM_SURG_TEXT; - } - - public void setTM_SURG_TEXT(String TM_SURG_TEXT) { - this.TM_SURG_TEXT = ClinicalValueUtil.defaultWithNA(TM_SURG_TEXT); - } - - public String getTM_OVERRIDE_COM() { - return TM_OVERRIDE_COM; - } - - public void setTM_OVERRIDE_COM(String TM_OVERRIDE_COM) { - this.TM_OVERRIDE_COM = ClinicalValueUtil.defaultWithNA(TM_OVERRIDE_COM); - } - - public String getTM_CSSIZE() { - return TM_CSSIZE; - } - - public void setTM_CSSIZE(String TM_CSSIZE) { - this.TM_CSSIZE = ClinicalValueUtil.defaultWithNA(TM_CSSIZE); - } - - public String getTM_CSEXT() { - return TM_CSEXT; - } - - public void setTM_CSEXT(String TM_CSEXT) { - this.TM_CSEXT = ClinicalValueUtil.defaultWithNA(TM_CSEXT); - } - - public String getTM_CSEXTEV() { - return TM_CSEXTEV; - } - - public void setTM_CSEXTEV(String TM_CSEXTEV) { - this.TM_CSEXTEV = ClinicalValueUtil.defaultWithNA(TM_CSEXTEV); - } - - public String getTM_CSLMND() { - return TM_CSLMND; - } - - public void setTM_CSLMND(String TM_CSLMND) { - this.TM_CSLMND = ClinicalValueUtil.defaultWithNA(TM_CSLMND); - } - - public String getTM_CSRGNEV() { - return TM_CSRGNEV; - } - - public void setTM_CSRGNEV(String TM_CSRGNEV) { - this.TM_CSRGNEV = ClinicalValueUtil.defaultWithNA(TM_CSRGNEV); - } - - public String getTM_CSMETDX() { - return TM_CSMETDX; - } - - public void setTM_CSMETDX(String TM_CSMETDX) { - this.TM_CSMETDX = ClinicalValueUtil.defaultWithNA(TM_CSMETDX); - } - - public String getTM_CSMETEV() { - return TM_CSMETEV; - } - - public void setTM_CSMETEV(String TM_CSMETEV) { - this.TM_CSMETEV = ClinicalValueUtil.defaultWithNA(TM_CSMETEV); - } - - public String getTM_TSTAGE() { - return TM_TSTAGE; - } - - public void setTM_TSTAGE(String TM_TSTAGE) { - this.TM_TSTAGE = ClinicalValueUtil.defaultWithNA(TM_TSTAGE); - } - - public String getTM_TSTAGE_DESC() { - return TM_TSTAGE_DESC; - } - - public void setTM_TSTAGE_DESC(String TM_TSTAGE_DESC) { - this.TM_TSTAGE_DESC = ClinicalValueUtil.defaultWithNA(TM_TSTAGE_DESC); - } - - public String getTM_NSTAGE() { - return TM_NSTAGE; - } - - public void setTM_NSTAGE(String TM_NSTAGE) { - this.TM_NSTAGE = ClinicalValueUtil.defaultWithNA(TM_NSTAGE); - } - - public String getTM_NSTAGE_DESC() { - return TM_NSTAGE_DESC; - } - - public void setTM_NSTAGE_DESC(String TM_NSTAGE_DESC) { - this.TM_NSTAGE_DESC = ClinicalValueUtil.defaultWithNA(TM_NSTAGE_DESC); - } - - public String getTM_MSTAGE() { - return TM_MSTAGE; - } - - public void setTM_MSTAGE(String TM_MSTAGE) { - this.TM_MSTAGE = ClinicalValueUtil.defaultWithNA(TM_MSTAGE); - } - - public String getTM_MSTAGE_DESC() { - return TM_MSTAGE_DESC; - } - - public void setTM_MSTAGE_DESC(String TM_MSTAGE_DESC) { - this.TM_MSTAGE_DESC = ClinicalValueUtil.defaultWithNA(TM_MSTAGE_DESC); - } - - public String getTM_TBASIS() { - return TM_TBASIS; - } - - public void setTM_TBASIS(String TM_TBASIS) { - this.TM_TBASIS = ClinicalValueUtil.defaultWithNA(TM_TBASIS); - } - - public String getTM_TBASIS_DESC() { - return TM_TBASIS_DESC; - } - - public void setTM_TBASIS_DESC(String TM_TBASIS_DESC) { - this.TM_TBASIS_DESC = ClinicalValueUtil.defaultWithNA(TM_TBASIS_DESC); - } - - public String getTM_NBASIS() { - return TM_NBASIS; - } - - public void setTM_NBASIS(String TM_NBASIS) { - this.TM_NBASIS = ClinicalValueUtil.defaultWithNA(TM_NBASIS); - } - - public String getTM_NBASIS_DESC() { - return TM_NBASIS_DESC; - } - - public void setTM_NBASIS_DESC(String TM_NBASIS_DESC) { - this.TM_NBASIS_DESC = ClinicalValueUtil.defaultWithNA(TM_NBASIS_DESC); - } - - public String getTM_MBASIS() { - return TM_MBASIS; - } - - public void setTM_MBASIS(String TM_MBASIS) { - this.TM_MBASIS = ClinicalValueUtil.defaultWithNA(TM_MBASIS); - } - - public String getTM_MBASIS_DESC() { - return TM_MBASIS_DESC; - } - - public void setTM_MBASIS_DESC(String TM_MBASIS_DESC) { - this.TM_MBASIS_DESC = ClinicalValueUtil.defaultWithNA(TM_MBASIS_DESC); - } - - public String getTM_AJCC() { - return TM_AJCC; - } - - public void setTM_AJCC(String TM_AJCC) { - this.TM_AJCC = ClinicalValueUtil.defaultWithNA(TM_AJCC); - } - - public String getTM_AJCC_DESC() { - return TM_AJCC_DESC; - } - - public void setTM_AJCC_DESC(String TM_AJCC_DESC) { - this.TM_AJCC_DESC = ClinicalValueUtil.defaultWithNA(TM_AJCC_DESC); - } - - public String getTM_MSK_STG() { - return TM_MSK_STG; - } - - public void setTM_MSK_STG(String TM_MSK_STG) { - this.TM_MSK_STG = ClinicalValueUtil.defaultWithNA(TM_MSK_STG); - } - - public Integer getAGE_AT_LAST_KNOWN_ALIVE_IN_DAYS() { - return AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS; - } - - public void setAGE_AT_LAST_KNOWN_ALIVE_IN_DAYS(Integer AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS) { - this.AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS = AGE_AT_LAST_KNOWN_ALIVE_IN_DAYS; - } - - public Integer getAGE_AT_DATE_OF_DEATH_IN_DAYS() { - return AGE_AT_DATE_OF_DEATH_IN_DAYS; - } - - public void setAGE_AT_DATE_OF_DEATH_IN_DAYS(Integer AGE_AT_DATE_OF_DEATH_IN_DAYS) { - this.AGE_AT_DATE_OF_DEATH_IN_DAYS = AGE_AT_DATE_OF_DEATH_IN_DAYS; - } - - @Override - public String toString(){ - return ToStringBuilder.reflectionToString(this); - } - - public List getFieldNames(){ - List fieldNames = new ArrayList<>(); - fieldNames.add("PT_ID_ICDO"); - fieldNames.add("DMP_ID_ICDO"); - fieldNames.add("TM_TUMOR_SEQ_DESC"); - fieldNames.add("TM_ACC_YEAR"); - fieldNames.add("TM_FIRST_MSK_YEAR"); - fieldNames.add("AGE_AT_TM_FIRST_MSK_DATE_IN_DAYS"); - fieldNames.add("TM_CASE_STS"); - fieldNames.add("TM_CASE_STS_DESC"); - fieldNames.add("TM_CASE_EFF_YEAR"); - fieldNames.add("AGE_AT_TM_CASE_EFF_DATE_IN_DAYS"); - fieldNames.add("TM_STATE_AT_DX"); - fieldNames.add("TM_SMOKING_HX"); - fieldNames.add("TM_SMOKING_HX_DESC"); - fieldNames.add("TM_OCCUPATION"); - fieldNames.add("TM_OCCUPATION_DESC"); - fieldNames.add("TM_FACILITY_FROM"); - fieldNames.add("FACILITY_FROM_DESC"); - fieldNames.add("TM_FACILITY_TO"); - fieldNames.add("FACILITY_TO_DESC"); - fieldNames.add("TM_DX_YEAR"); - fieldNames.add("AGE_AT_TM_DX_DATE_IN_DAYS"); - fieldNames.add("TM_SITE_CD"); - fieldNames.add("TM_SITE_DESC"); - fieldNames.add("TM_LATERALITY_CD"); - fieldNames.add("TM_LATERALITY_DESC"); - fieldNames.add("TM_HIST_CD"); - fieldNames.add("TM_HIST_DESC"); - fieldNames.add("TM_DX_CONFRM_CD"); - fieldNames.add("TM_DX_CONFRM_DESC"); - fieldNames.add("TM_REGNODE_EXM_NO"); - fieldNames.add("TM_REGNODE_POS_NO"); - fieldNames.add("TM_TUMOR_SIZE"); - fieldNames.add("TM_RESID_TUMOR_CD"); - fieldNames.add("TM_RESID_TUMOR_DESC"); - fieldNames.add("TM_GENERAL_STG"); - fieldNames.add("TM_GENERAL_STG_DESC"); - fieldNames.add("TM_TNM_EDITION"); - fieldNames.add("TM_TNM_EDITION_DESC"); - fieldNames.add("TM_CLIN_TNM_T"); - fieldNames.add("TM_CLIN_TNM_T_DESC"); - fieldNames.add("TM_CLIN_TNM_N"); - fieldNames.add("TM_CLIN_TNM_N_DESC"); - fieldNames.add("TM_CLIN_TNM_M"); - fieldNames.add("TM_CLIN_TNM_M_DESC"); - fieldNames.add("TM_CLIN_STG_GRP"); - fieldNames.add("TM_PATH_TNM_T"); - fieldNames.add("TM_PATH_TNM_T_DESC"); - fieldNames.add("TM_PATH_TNM_N"); - fieldNames.add("TM_PATH_TNM_N_DESC"); - fieldNames.add("TM_PATH_TNM_M"); - fieldNames.add("TM_PATH_TNM_M_DESC"); - fieldNames.add("TM_PATH_STG_GRP"); - fieldNames.add("TM_PATH_RPT_AV"); - fieldNames.add("TM_CA_STS_AT_ACC"); - fieldNames.add("TM_FIRST_RECUR_YEAR"); - fieldNames.add("AGE_AT_TM_FIRST_RECUR_DATE_IN_DAYS"); - fieldNames.add("TM_FIRST_RECUR_TYP"); - fieldNames.add("TM_ADM_YEAR"); - fieldNames.add("AGE_AT_TM_ADM_DATE_IN_DAYS"); - fieldNames.add("TM_DSCH_YEAR"); - fieldNames.add("AGE_AT_TM_DSCH_DATE_IN_DAYS"); - fieldNames.add("TM_SURG_DSCH_YEAR"); - fieldNames.add("AGE_AT_TM_SURG_DSCH_DATE_IN_DAYS"); - fieldNames.add("TM_READM_WTHN_30D"); - fieldNames.add("TM_FIRST_TX_YEAR"); - fieldNames.add("AGE_AT_TM_FIRST_TX_DATE_IN_DAYS"); - fieldNames.add("TM_NON_CA_SURG_SUM"); - fieldNames.add("TM_NON_CA_SURG_SUM_DESC"); - fieldNames.add("TM_NON_CA_SURG_MSK"); - fieldNames.add("TM_NON_CA_SURG_YEAR"); - fieldNames.add("AGE_AT_TM_NON_CA_SURG_DATE_IN_DAYS"); - fieldNames.add("TM_CA_SURG_98"); - fieldNames.add("TM_CA_SURG_98_MSK"); - fieldNames.add("TM_CA_SURG_03"); - fieldNames.add("TM_CA_SURG_03_MSK"); - fieldNames.add("TM_OTH_SURG_98"); - fieldNames.add("TM_OTH_SURG_98_MSK"); - fieldNames.add("TM_OTH_SURG_03"); - fieldNames.add("TM_OTH_SURG_03_MSK"); - fieldNames.add("TM_OTH_SURG_CD"); - fieldNames.add("TM_OTH_SURG_CD_DESC"); - fieldNames.add("TM_RGN_SCOP_98"); - fieldNames.add("TM_RGN_SCOP_98_MSK"); - fieldNames.add("TM_RGN_SCOP_03"); - fieldNames.add("TM_RGN_SCOP_03_MSK"); - fieldNames.add("TM_REGNODE_SCOP_CD"); - fieldNames.add("TM_REGNODE_SCOP_DESC"); - fieldNames.add("TM_RECON_SURG"); - fieldNames.add("TM_RECON_SURG_DESC"); - fieldNames.add("TM_CA_SURG_YEAR"); - fieldNames.add("AGE_AT_TM_CA_SURG_DATE_IN_DAYS"); - fieldNames.add("TM_SURG_DEF_YEAR"); - fieldNames.add("AGE_AT_TM_SURG_DEF_DATE_IN_DAYS"); - fieldNames.add("TM_REASON_NO_SURG"); - fieldNames.add("REASON_NO_SURG_DESC"); - fieldNames.add("TM_PRIM_SURGEON"); - fieldNames.add("TM_PRIM_SURGEON_NAME"); - fieldNames.add("TM_ATN_DR_NO"); - fieldNames.add("TM_ATN_DR_NAME"); - fieldNames.add("TM_REASON_NO_RAD"); - fieldNames.add("TM_REASON_NO_RAD_DESC"); - fieldNames.add("TM_RAD_STRT_YEAR"); - fieldNames.add("AGE_AT_TM_RAD_STRT_DATE_IN_DAYS"); - fieldNames.add("TM_RAD_END_YEAR"); - fieldNames.add("AGE_AT_TM_RAD_END_DATE_IN_DAYS"); - fieldNames.add("TM_RAD_TX_MOD"); - fieldNames.add("TM_RAD_TX_MOD_DESC"); - fieldNames.add("TM_BOOST_RAD_MOD"); - fieldNames.add("TM_BOOST_RAD_MOD_DESC"); - fieldNames.add("TM_LOC_RAD_TX"); - fieldNames.add("TM_LOC_RAD_TX_DESC"); - fieldNames.add("TM_RAD_TX_VOL"); - fieldNames.add("TM_RAD_TX_VOL_DESC"); - fieldNames.add("TM_NUM_TX_THIS_VOL"); - fieldNames.add("TM_NUM_TX_THIS_VOL_DESC"); - fieldNames.add("TM_REG_RAD_DOSE"); - fieldNames.add("TM_REG_RAD_DOSE_DESC"); - fieldNames.add("TM_BOOST_RAD_DOSE"); - fieldNames.add("TM_BOOST_RAD_DOSE_DESC"); - fieldNames.add("TM_RAD_SURG_SEQ"); - fieldNames.add("TM_RAD_SURG_SEQ_DESC"); - fieldNames.add("TM_RAD_MD"); - fieldNames.add("TM_RAD_MD_NAME"); - fieldNames.add("TM_SYST_STRT_YEAR"); - fieldNames.add("AGE_AT_TM_SYST_STRT_DATE_IN_DAYS"); - fieldNames.add("TM_OTH_STRT_YEAR"); - fieldNames.add("AGE_AT_TM_OTH_STRT_DATE_IN_DAYS"); - fieldNames.add("TM_CHEM_SUM"); - fieldNames.add("TM_CHEM_SUM_DESC"); - fieldNames.add("TM_CHEM_SUM_MSK"); - fieldNames.add("TM_CHEM_SUM_MSK_DESC"); - fieldNames.add("TM_TUMOR_SEQ"); - fieldNames.add("TM_HORM_SUM"); - fieldNames.add("TM_HORM_SUM_DESC"); - fieldNames.add("TM_HORM_SUM_MSK"); - fieldNames.add("TM_HORM_SUM_MSK_DESC"); - fieldNames.add("TM_BRM_SUM"); - fieldNames.add("TM_BRM_SUM_DESC"); - fieldNames.add("TM_BRM_SUM_MSK"); - fieldNames.add("TM_BRM_SUM_MSK_DESC"); - fieldNames.add("TM_OTH_SUM"); - fieldNames.add("TM_OTH_SUM_DESC"); - fieldNames.add("TM_OTH_SUM_MSK"); - fieldNames.add("TM_OTH_SUM_MSK_DESC"); - fieldNames.add("TM_PALLIA_PROC"); - fieldNames.add("TM_PALLIA_PROC_DESC"); - fieldNames.add("TM_PALLIA_PROC_MSK"); - fieldNames.add("TM_PALLIA_PROC_MSK_DESC"); - fieldNames.add("TM_ONCOLOGY_MD"); - fieldNames.add("TM_ONCOLOGY_MD_NAME"); - fieldNames.add("TM_PRCS_YEAR"); - fieldNames.add("AGE_AT_TM_PRCS_DATE_IN_DAYS"); - fieldNames.add("TM_PATH_TEXT"); - fieldNames.add("TM_SURG_TEXT"); - fieldNames.add("TM_OVERRIDE_COM"); - fieldNames.add("TM_CSSIZE"); - fieldNames.add("TM_CSEXT"); - fieldNames.add("TM_CSEXTEV"); - fieldNames.add("TM_CSLMND"); - fieldNames.add("TM_CSRGNEV"); - fieldNames.add("TM_CSMETDX"); - fieldNames.add("TM_CSMETEV"); - fieldNames.add("TM_TSTAGE"); - fieldNames.add("TM_TSTAGE_DESC"); - fieldNames.add("TM_NSTAGE"); - fieldNames.add("TM_NSTAGE_DESC"); - fieldNames.add("TM_MSTAGE"); - fieldNames.add("TM_MSTAGE_DESC"); - fieldNames.add("TM_TBASIS"); - fieldNames.add("TM_TBASIS_DESC"); - fieldNames.add("TM_NBASIS"); - fieldNames.add("TM_NBASIS_DESC"); - fieldNames.add("TM_MBASIS"); - fieldNames.add("TM_MBASIS_DESC"); - fieldNames.add("TM_AJCC"); - fieldNames.add("TM_AJCC_DESC"); - fieldNames.add("TM_MSK_STG"); - return fieldNames; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCREthnicityMapping.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCREthnicityMapping.java deleted file mode 100644 index d7760407b..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCREthnicityMapping.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class NAACCREthnicityMapping { - private Integer NECM_CODE; - private String NECM_CBIOPORTAL_LABEL; - - public NAACCREthnicityMapping() {} - - public NAACCREthnicityMapping(Integer NECM_CODE, String NECM_CBIOPORTAL_LABEL) { - this.NECM_CODE = NECM_CODE; - this.NECM_CBIOPORTAL_LABEL = NECM_CBIOPORTAL_LABEL; - } - - /** - * @return the NECM_CODE - */ - public Integer getNECM_CODE() { - return NECM_CODE; - } - - /** - * @param NECM_CODE the NECM_CODE to set - */ - public void setNECM_CODE(Integer NECM_CODE) { - this.NECM_CODE = NECM_CODE; - } - - /** - * @return the NECM_CBIOPORTAL_LABEL - */ - public String getNECM_CBIOPORTAL_LABEL() { - return NECM_CBIOPORTAL_LABEL; - } - - /** - * @param NECM_CBIOPORTAL_LABEL the NECM_CBIOPORTAL_LABEL to set - */ - public void setNECM_CBIOPORTAL_LABEL(String NECM_CBIOPORTAL_LABEL) { - this.NECM_CBIOPORTAL_LABEL = NECM_CBIOPORTAL_LABEL; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRRaceMapping.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRRaceMapping.java deleted file mode 100644 index f1c9cdde5..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRRaceMapping.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class NAACCRRaceMapping { - private Integer NRCM_CODE; - private String NRCM_CBIOPORTAL_LABEL; - - public NAACCRRaceMapping() {} - - public NAACCRRaceMapping(Integer NRCM_CODE, String NRCM_CBIOPORTAL_LABEL) { - this.NRCM_CODE = NRCM_CODE; - this.NRCM_CBIOPORTAL_LABEL = NRCM_CBIOPORTAL_LABEL; - } - - /** - * @return the NRCM_CODE - */ - public Integer getNRCM_CODE() { - return NRCM_CODE; - } - - /** - * @param NRCM_CODE the NRCM_CODE to set - */ - public void setNRCM_CODE(Integer NRCM_CODE) { - this.NRCM_CODE = NRCM_CODE; - } - - /** - * @return the NRCM_CBIOPORTAL_LABEL - */ - public String getNRCM_CBIOPORTAL_LABEL() { - return NRCM_CBIOPORTAL_LABEL; - } - - /** - * @param NRCM_CBIOPORTAL_LABEL the NRCM_CBIOPORTAL_LABEL to set - */ - public void setNRCM_CBIOPORTAL_LABEL(String NRCM_CBIOPORTAL_LABEL) { - this.NRCM_CBIOPORTAL_LABEL = NRCM_CBIOPORTAL_LABEL; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRSexMapping.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRSexMapping.java deleted file mode 100644 index 2e68ba9d6..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/NAACCRSexMapping.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class NAACCRSexMapping { - private Integer NSCM_CODE; - private String NSCM_CBIOPORTAL_LABEL; - - public NAACCRSexMapping() {} - - public NAACCRSexMapping(Integer NSCM_CODE, String NSCM_CBIOPORTAL_LABEL) { - this.NSCM_CODE = NSCM_CODE; - this.NSCM_CBIOPORTAL_LABEL = NSCM_CBIOPORTAL_LABEL; - } - - /** - * @return the NSCM_CODE - */ - public Integer getNSCM_CODE() { - return NSCM_CODE; - } - - /** - * @param NSCM_CODE the NSCM_CODE to set - */ - public void setNSCM_CODE(Integer NSCM_CODE) { - this.NSCM_CODE = NSCM_CODE; - } - - /** - * @return the NSCM_CBIOPORTAL_LABEL - */ - public String getNSCM_CBIOPORTAL_LABEL() { - return NSCM_CBIOPORTAL_LABEL; - } - - /** - * @param NSCM_CBIOPORTAL_LABEL the NSCM_CBIOPORTAL_LABEL to set - */ - public void setNSCM_CBIOPORTAL_LABEL(String NSCM_CBIOPORTAL_LABEL) { - this.NSCM_CBIOPORTAL_LABEL = NSCM_CBIOPORTAL_LABEL; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalCompositeRecord.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalCompositeRecord.java deleted file mode 100644 index 1659e565a..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalCompositeRecord.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalCompositeRecord { - - private Skcm_mskcc_2015_chantNormalizedClinicalRecord record; - private String sampleRecord; - private String patientRecord; - - public Skcm_mskcc_2015_chantClinicalCompositeRecord() {} - public Skcm_mskcc_2015_chantClinicalCompositeRecord(Skcm_mskcc_2015_chantNormalizedClinicalRecord record) { - this.record = record; - } - - public Skcm_mskcc_2015_chantNormalizedClinicalRecord getRecord() { - return this.record; - } - - public void setRecord(Skcm_mskcc_2015_chantNormalizedClinicalRecord record) { - this.record = record; - } - - public String getSampleRecord() { - return sampleRecord; - } - - public void setSampleRecord(String sampleRecord) { - this.sampleRecord = sampleRecord; - } - - public String getPatientRecord() { - return patientRecord; - } - - public void setPatientRecord(String patientRecord) { - this.patientRecord = patientRecord; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalRecord.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalRecord.java deleted file mode 100644 index 9d5aadd1b..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantClinicalRecord.java +++ /dev/null @@ -1,974 +0,0 @@ -/* - * Copyright (c) 2016, 2018, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalRecord { - private String patientId; - private String sampleId; - private String melspcPtid; - private String melspcStageYear; - private String melspcStgGrpName; - private String melgPtid; - private String melgStsDesc; - private String melgStsSrcDesc; - private String melgActvStsDesc; - private String melgDermagrphxDesc; - private String melgPresStgYear; - private String melgFamilyHxDesc; - private String melg1stRecurYear; - private String melgLocalDesc; - private String melgNodalDesc; - private String melgIntransitDesc; - private String melgSysDesc; - private String melgRecurNdszDesc; - private String melgRecurNodalNo; - private String melgLdh; - private String melgLdhYear; - private String melgMetsDesc; - private String melgAdjvntTxDesc; - private String melgSysTxDesc; - private String melgRadTxDesc; - private String melgSurgDesc; - private String melgTissueBankAvail; - private String melpPtid; - private String melpPrimSeq; - private String melpDxYear; - private String melpMskReviewDesc; - private String melpThicknessMm; - private String melpClarkLvlDesc; - private String melpUlcerationDesc; - private String melpSiteDesc; - private String melpSubSiteDesc; - private String melpTilsDesc; - private String melpRegressionDesc; - private String melpMarginsDesc; - private String melpMitidxUnkDesc; - private String melpHistTypeDesc; - private String melpSatellitesDesc; - private String melpExtSlidesDesc; - private String melpLnorgDxDesc; - private String melpLnclinStsDesc; - private String melpLnsentinbxDesx; - private String melpLnsentinbxYear; - private String melpLnprolysctDesc; - private String melpLnprosuccDesc; - private String melpLndsctCmpDesc; - private String melpLndsctYear; - private String melpLnmattedDesc; - private String LnextnodstDesc; - private String LnintrmetsDesc; - private String melpLnsize; - private String melpLnsizeUnkDesc; - private String melpLnslnlargSize; - private String melpLnihcDesc; - private String melpLnimmS100Desc; - private String melpLnimmhmb45Desc; - private String melpLnimmMelaDesc; - private String meliPtid; - private String meliDmpPatientId; - private String meliDmpSampleId; - private String meliReportYear; - private String meliProcedureYear; - private String meliTumorType; - private String meliPrimarySite; - private String meliMetSite; - private String melmsPtid; - private String melmsSiteTypeDesc; - private String melmsSiteDesc; - private String melmsSiteYear; - - private Map additionalProperties; - - public Skcm_mskcc_2015_chantClinicalRecord() { - } - - public Skcm_mskcc_2015_chantClinicalRecord(String melspcPtid, - String melspcStageYear, - String melspcStgGrpName, - String melgPtid, - String melgStsDesc, - String melgStsSrcDesc, - String melgActvStsDesc, - String melgDermagrphxDesc, - String melgPresStgYear, - String melgFamilyHxDesc, - String melg1stRecurYear, - String melgLocalDesc, - String melgNodalDesc, - String melgIntransitDesc, - String melgSysDesc, - String melgRecurNdszDesc, - String melgRecurNodalNo, - String melgLdh, - String melgLdhYear, - String melgMetsDesc, - String melgAdjvntTxDesc, - String melgSysTxDesc, - String melgRadTxDesc, - String melgSurgDesc, - String melgTissueBankAvail, - String melpPtid, - String melpPrimSeq, - String melpDxYear, - String melpMskReviewDesc, - String melpThicknessMm, - String melpClarkLvlDesc, - String melpUlcerationDesc, - String melpSiteDesc, - String melpSubSiteDesc, - String melpTilsDesc, - String melpRegressionDesc, - String melpMarginsDesc, - String melpMitidxUnkDesc, - String melpHistTypeDesc, - String melpSatellitesDesc, - String melpExtSlidesDesc, - String melpLnorgDxDesc, - String melpLnclinStsDesc, - String melpLnsentinbxDesx, - String melpLnsentinbxYear, - String melpLnprolysctDesc, - String melpLnprosuccDesc, - String melpLndsctCmpDesc, - String melpLndsctYear, - String melpLnmattedDesc, - String LnextnodstDesc, - String LnintrmetsDesc, - String melpLnsize, - String melpLnsizeUnkDesc, - String melpLnslnlargSize, - String melpLnihcDesc, - String melpLnimmS100Desc, - String melpLnimmhmb45Desc, - String melpLnimmMelaDesc, - String meliPtid, - String meliDmpPatientId, - String meliDmpSampleId, - String meliReportYear, - String meliProcedureYear, - String meliTumorType, - String meliPrimarySite, - String meliMetSite, - String melmsPtid, - String melmsSiteTypeDesc, - String melmsSiteDesc, - String melmsSiteYear) { - this.melspcPtid = ClinicalValueUtil.defaultWithNA(melspcPtid); - this.melspcStageYear = ClinicalValueUtil.defaultWithNA(melspcStageYear); - this.melspcStgGrpName = ClinicalValueUtil.defaultWithNA(melspcStgGrpName); - this.melgPtid = ClinicalValueUtil.defaultWithNA(melgPtid); - this.melgStsDesc = ClinicalValueUtil.defaultWithNA(melgStsDesc); - this.melgStsSrcDesc = ClinicalValueUtil.defaultWithNA(melgStsSrcDesc); - this.melgActvStsDesc = ClinicalValueUtil.defaultWithNA(melgActvStsDesc); - this.melgDermagrphxDesc = ClinicalValueUtil.defaultWithNA(melgDermagrphxDesc); - this.melgPresStgYear = ClinicalValueUtil.defaultWithNA(melgPresStgYear); - this.melgFamilyHxDesc = ClinicalValueUtil.defaultWithNA(melgFamilyHxDesc); - this.melg1stRecurYear = ClinicalValueUtil.defaultWithNA(melg1stRecurYear); - this.melgLocalDesc = ClinicalValueUtil.defaultWithNA(melgLocalDesc); - this.melgNodalDesc = ClinicalValueUtil.defaultWithNA(melgNodalDesc); - this.melgIntransitDesc = ClinicalValueUtil.defaultWithNA(melgIntransitDesc); - this.melgSysDesc = ClinicalValueUtil.defaultWithNA(melgSysDesc); - this.melgRecurNdszDesc = ClinicalValueUtil.defaultWithNA(melgRecurNdszDesc); - this.melgRecurNodalNo = ClinicalValueUtil.defaultWithNA(melgRecurNodalNo); - this.melgLdh = ClinicalValueUtil.defaultWithNA(melgLdh); - this.melgLdhYear = ClinicalValueUtil.defaultWithNA(melgLdhYear); - this.melgMetsDesc = ClinicalValueUtil.defaultWithNA(melgMetsDesc); - this.melgAdjvntTxDesc = ClinicalValueUtil.defaultWithNA(melgAdjvntTxDesc); - this.melgSysTxDesc = ClinicalValueUtil.defaultWithNA(melgSysTxDesc); - this.melgRadTxDesc = ClinicalValueUtil.defaultWithNA(melgRadTxDesc); - this.melgSurgDesc = ClinicalValueUtil.defaultWithNA(melgSurgDesc); - this.melgTissueBankAvail = ClinicalValueUtil.defaultWithNA(melgTissueBankAvail); - this.melpPtid = ClinicalValueUtil.defaultWithNA(melpPtid); - this.melpPrimSeq = ClinicalValueUtil.defaultWithNA(melpPrimSeq); - this.melpDxYear = ClinicalValueUtil.defaultWithNA(melpDxYear); - this.melpMskReviewDesc = ClinicalValueUtil.defaultWithNA(melpMskReviewDesc); - this.melpThicknessMm = ClinicalValueUtil.defaultWithNA(melpThicknessMm); - this.melpClarkLvlDesc = ClinicalValueUtil.defaultWithNA(melpClarkLvlDesc); - this.melpUlcerationDesc = ClinicalValueUtil.defaultWithNA(melpUlcerationDesc); - this.melpSiteDesc = ClinicalValueUtil.defaultWithNA(melpSiteDesc); - this.melpSubSiteDesc = ClinicalValueUtil.defaultWithNA(melpSubSiteDesc); - this.melpTilsDesc = ClinicalValueUtil.defaultWithNA(melpTilsDesc); - this.melpRegressionDesc = ClinicalValueUtil.defaultWithNA(melpRegressionDesc); - this.melpMarginsDesc = ClinicalValueUtil.defaultWithNA(melpMarginsDesc); - this.melpMitidxUnkDesc = ClinicalValueUtil.defaultWithNA(melpMitidxUnkDesc); - this.melpHistTypeDesc = ClinicalValueUtil.defaultWithNA(melpHistTypeDesc); - this.melpSatellitesDesc = ClinicalValueUtil.defaultWithNA(melpSatellitesDesc); - this.melpExtSlidesDesc = ClinicalValueUtil.defaultWithNA(melpExtSlidesDesc); - this.melpLnorgDxDesc = ClinicalValueUtil.defaultWithNA(melpLnorgDxDesc); - this.melpLnclinStsDesc = ClinicalValueUtil.defaultWithNA(melpLnclinStsDesc); - this.melpLnsentinbxDesx = ClinicalValueUtil.defaultWithNA(melpLnsentinbxDesx); - this.melpLnsentinbxYear = ClinicalValueUtil.defaultWithNA(melpLnsentinbxYear); - this.melpLnprolysctDesc = ClinicalValueUtil.defaultWithNA(melpLnprolysctDesc); - this.melpLnprosuccDesc = ClinicalValueUtil.defaultWithNA(melpLnprosuccDesc); - this.melpLndsctCmpDesc = ClinicalValueUtil.defaultWithNA(melpLndsctCmpDesc); - this.melpLndsctYear = ClinicalValueUtil.defaultWithNA(melpLndsctYear); - this.melpLnmattedDesc = ClinicalValueUtil.defaultWithNA(melpLnmattedDesc); - this.LnextnodstDesc = ClinicalValueUtil.defaultWithNA(LnextnodstDesc); - this.LnintrmetsDesc = ClinicalValueUtil.defaultWithNA(LnintrmetsDesc); - this.melpLnsize = ClinicalValueUtil.defaultWithNA(melpLnsize); - this.melpLnsizeUnkDesc = ClinicalValueUtil.defaultWithNA(melpLnsizeUnkDesc); - this.melpLnslnlargSize = ClinicalValueUtil.defaultWithNA(melpLnslnlargSize); - this.melpLnihcDesc = ClinicalValueUtil.defaultWithNA(melpLnihcDesc); - this.melpLnimmS100Desc = ClinicalValueUtil.defaultWithNA(melpLnimmS100Desc); - this.melpLnimmhmb45Desc = ClinicalValueUtil.defaultWithNA(melpLnimmhmb45Desc); - this.melpLnimmMelaDesc = ClinicalValueUtil.defaultWithNA(melpLnimmMelaDesc); - this.meliPtid = ClinicalValueUtil.defaultWithNA(meliPtid); - this.meliDmpPatientId = ClinicalValueUtil.defaultWithNA(meliDmpPatientId); - this.meliDmpSampleId = ClinicalValueUtil.defaultWithNA(meliDmpSampleId); - this.meliReportYear = ClinicalValueUtil.defaultWithNA(meliReportYear); - this.meliProcedureYear = ClinicalValueUtil.defaultWithNA(meliProcedureYear); - this.meliTumorType = ClinicalValueUtil.defaultWithNA(meliTumorType); - this.meliPrimarySite = ClinicalValueUtil.defaultWithNA(meliPrimarySite); - this.meliMetSite = ClinicalValueUtil.defaultWithNA(meliMetSite); - this.melmsPtid = ClinicalValueUtil.defaultWithNA(melmsPtid); - this.melmsSiteTypeDesc = ClinicalValueUtil.defaultWithNA(melmsSiteTypeDesc); - this.melmsSiteDesc = ClinicalValueUtil.defaultWithNA(melmsSiteDesc); - this.melmsSiteYear = ClinicalValueUtil.defaultWithNA(melmsSiteYear); - } - - public String getPATIENT_ID() { - if (patientId != null) { - return patientId; - } - if (!melspcPtid.equals("NA")) { - return melspcPtid; - } - if (!melgPtid.equals("NA")) { - return melgPtid; - } - if (!melpPtid.equals("NA")) { - return melpPtid; - } - return melmsPtid; - } - - public void setPATIENT_ID(String patientId) { - this.patientId = patientId; - } - - public String getSAMPLE_ID() { - if (sampleId != null) { - return sampleId; - } - if (!meliDmpSampleId.equals("NA")) { - return meliDmpSampleId; - } - if (!melpPrimSeq.equals("NA")) { - return melpPtid + "_" + melpPrimSeq; - } - if (!melspcPtid.equals("NA")) { - return melspcPtid; - } - if (!melgPtid.equals("NA")) { - return melgPtid; - } - return melmsPtid; - } - - public void setSAMPLE_ID(String sampleId) { - this.sampleId = sampleId; - } - - public String getMELSPC_PTID() { - return melspcPtid; - } - - public void setMELSPC_PTID(String melspcPtid) { - this.melspcPtid = melspcPtid; - } - - public String getMELSPC_STAGE_YEAR() { - return melspcStageYear; - } - - public void setMELSPC_STAGE_YEAR(String melspcStageYear) { - this.melspcStageYear = melspcStageYear; - } - - public String getMELSPC_STG_GRP_NAME() { - return melspcStgGrpName; - } - - public void setMELSPC_STG_GRP_NAME(String melspcStgGrpName) { - this.melspcStgGrpName = melspcStgGrpName; - } - - public String getMELG_PTID() { - return melgPtid; - } - - public void setMELG_PTID(String melgPtid) { - this.melgPtid = melgPtid; - } - - public String getMELG_STS_DESC() { - return melgStsDesc; - } - - public void setMELG_STS_DESC(String melgStsDesc) { - this.melgStsDesc = melgStsDesc; - } - - public String getMELG_STS_SRCE_DESC() { - return melgStsSrcDesc; - } - - public void setMELG_STS_SRCE_DESC(String melgStsSrcDesc) { - this.melgStsSrcDesc = melgStsSrcDesc; - } - - public String getMELG_ACTV_STS_DESC() { - return melgActvStsDesc; - } - - public void setMELG_ACTV_STS_DESC(String melgActvStsDesc) { - this.melgActvStsDesc = melgActvStsDesc; - } - - public String getMELG_DERMAGRPHX_DESC() { - return melgDermagrphxDesc; - } - - public void setMELG_DERMAGRPHX_DESC(String melgDermagrphxDesc) { - this.melgDermagrphxDesc = melgDermagrphxDesc; - } - - public String getMELG_PRES_STG_YEAR() { - return melgPresStgYear; - } - - public void setMELG_PRES_STG_YEAR(String melgPresStgYear) { - this.melgPresStgYear = melgPresStgYear; - } - - public String getMELG_FAMILY_HX_DESC() { - return melgFamilyHxDesc; - } - - public void setMELG_FAMILY_HX_DESC(String melgFamilyHxDesc) { - this.melgFamilyHxDesc = melgFamilyHxDesc; - } - - public String getMELG_1ST_RECUR_YEAR() { - return melg1stRecurYear; - } - - public void setMELG_1ST_RECUR_YEAR(String melg1stRecurYear) { - this.melg1stRecurYear = melg1stRecurYear; - } - - public String getMELG_LOCAL_DESC() { - return melgLocalDesc; - } - - public void setMELG_LOCAL_DESC(String melgLocalDesc) { - this.melgLocalDesc = melgLocalDesc; - } - - public String getMELG_NODAL_DESC() { - return melgNodalDesc; - } - - public void setMELG_NODAL_DESC(String melgNodalDesc) { - this.melgNodalDesc = melgNodalDesc; - } - - public String getMELG_INTRANSIT_DESC() { - return melgIntransitDesc; - } - - public void setMELG_INTRANSIT_DESC(String melgIntransitDesc) { - this.melgIntransitDesc = melgIntransitDesc; - } - - public String getMELG_SYS_DESC() { - return melgSysDesc; - } - - public void setMELG_SYS_DESC(String melgSysDesc) { - this.melgSysDesc = melgSysDesc; - } - - public String getMELG_RECUR_NDSZ_DESC() { - return melgRecurNdszDesc; - } - - public void setMELG_RECUR_NDSZ_DESC(String melgRecurNdszDesc) { - this.melgRecurNdszDesc = melgRecurNdszDesc; - } - - public String getMELG_RECUR_NODAL_NO() { - return melgRecurNodalNo; - } - - public void setMELG_RECUR_NODAL_NO(String melgRecurNodalNo) { - this.melgRecurNodalNo = melgRecurNodalNo; - } - - public String getMELG_LDH() { - return melgLdh; - } - - public void setMELG_LDH(String melgLdh) { - this.melgLdh = melgLdh; - } - - public String getMELG_LDH_YEAR() { - return melgLdhYear; - } - - public void setMELG_LDH_YEAR(String melgLdhYear) { - this.melgLdhYear = melgLdhYear; - } - - public String getMELG_METS_DESC() { - return melgMetsDesc; - } - - public void setMELG_METS_DESC(String melgMetsDesc) { - this.melgMetsDesc = melgMetsDesc; - } - - public String getMELG_ADJVNT_TX_DESC() { - return melgAdjvntTxDesc; - } - - public void setMELG_ADJVNT_TX_DESC(String melgAdjvntTxDesc) { - this.melgAdjvntTxDesc = melgAdjvntTxDesc; - } - - public String getMELG_SYS_TX_DESC() { - return melgSysTxDesc; - } - - public void setMELG_SYS_TX_DESC(String melgSysTxDesc) { - this.melgSysTxDesc = melgSysTxDesc; - } - - public String getMELG_RAD_TX_DESC() { - return melgRadTxDesc; - } - - public void setMELG_RAD_TX_DESC(String melgRadTxDesc) { - this.melgRadTxDesc = melgRadTxDesc; - } - - public String getMELG_SURG_DESC() { - return melgSurgDesc; - } - - public void setMELG_SURG_DESC(String melgSurgDesc) { - this.melgSurgDesc = melgSurgDesc; - } - - public String getMELG_TISSUE_BANK_AVAIL() { - return melgTissueBankAvail; - } - - public void setMELG_TISSUE_BANK_AVAIL(String melgTissueBankAvail) { - this.melgTissueBankAvail = melgTissueBankAvail; - } - - public String getMELP_PTID() { - return melpPtid; - } - - public void setMELP_PTID(String melpPtid) { - this.melpPtid = melpPtid; - } - - public String getMELP_PRIM_SEQ() { - return melpPrimSeq; - } - - public void setMELP_PRIM_SEQ(String melpPrimSeq) { - this.melpPrimSeq = melpPrimSeq; - } - - public String getMELP_DX_YEAR() { - return melpDxYear; - } - - public void setMELP_DX_YEAR(String melpDxYear) { - this.melpDxYear = melpDxYear; - } - - public String getMELP_MSK_REVIEW_DESC() { - return melpMskReviewDesc; - } - - public void setMELP_MSK_REVIEW_DESC(String melpMskReviewDesc) { - this.melpMskReviewDesc = melpMskReviewDesc; - } - - public String getMELP_THICKNESS_MM() { - return melpThicknessMm; - } - - public void setMELP_THICKNESS_MM(String melpThicknessMm) { - this.melpThicknessMm = melpThicknessMm; - } - - public String getMELP_CLARK_LVL_DESC() { - return melpClarkLvlDesc; - } - - public void setMELP_CLARK_LVL_DESC(String melpClarkLvlDesc) { - this.melpClarkLvlDesc = melpClarkLvlDesc; - } - - public String getMELP_ULCERATION_DESC() { - return melpUlcerationDesc; - } - - public void setMELP_ULCERATION_DESC(String melpUlcerationDesc) { - this.melpUlcerationDesc = melpUlcerationDesc; - } - - public String getMELP_SITE_DESC() { - return melpSiteDesc; - } - - public void setMELP_SITE_DESC(String melpSiteDesc) { - this.melpSiteDesc = melpSiteDesc; - } - - public String getMELP_SUB_SITE_DESC() { - return melpSubSiteDesc; - } - - public void setMELP_SUB_SITE_DESC(String melpSubSiteDesc) { - this.melpSubSiteDesc = melpSubSiteDesc; - } - - public String getMELP_TILS_DESC() { - return melpTilsDesc; - } - - public void setMELP_TILS_DESC(String melpTilsDesc) { - this.melpTilsDesc = melpTilsDesc; - } - - public String getMELP_REGRESSION_DESC() { - return melpRegressionDesc; - } - - public void setMELP_REGRESSION_DESC(String melpRegressionDesc) { - this.melpRegressionDesc = melpRegressionDesc; - } - - public String getMELP_MARGINS_DESC() { - return melpMarginsDesc; - } - - public void setMELP_MARGINS_DESC(String melpMarginsDesc) { - this.melpMarginsDesc = melpMarginsDesc; - } - - public String getMELP_MITIDX_UNK_DESC() { - return melpMitidxUnkDesc; - } - - public void setMELP_MITIDX_UNK_DESC(String melpMitidxUnkDesc) { - this.melpMitidxUnkDesc = melpMitidxUnkDesc; - } - - public String getMELP_HIST_TYPE_DESC() { - return melpHistTypeDesc; - } - - public void setMELP_HIST_TYPE_DESC(String melpHistTypeDesc) { - this.melpHistTypeDesc = melpHistTypeDesc; - } - - public String getMELP_SATELLITES_DESC() { - return melpSatellitesDesc; - } - - public void setMELP_SATELLITES_DESC(String melpSatellitesDesc) { - this.melpSatellitesDesc = melpSatellitesDesc; - } - - public String getMELP_EXT_SLIDES_DESC() { - return melpExtSlidesDesc; - } - - public void setMELP_EXT_SLIDES_DESC(String melpExtSlidesDesc) { - this.melpExtSlidesDesc = melpExtSlidesDesc; - } - - public String getMELP_LNORG_DX_DESC() { - return melpLnorgDxDesc; - } - - public void setMELP_LNORG_DX_DESC(String melpLnorgDxDesc) { - this.melpLnorgDxDesc = melpLnorgDxDesc; - } - - public String getMELP_LNCLIN_STS_DESC() { - return melpLnclinStsDesc; - } - - public void setMELP_LNCLIN_STS_DESC(String melpLnclinStsDesc) { - this.melpLnclinStsDesc = melpLnclinStsDesc; - } - - public String getMELP_LNSENTINBX_DESC() { - return melpLnsentinbxDesx; - } - - public void setMELP_LNSENTINBX_DESC(String melpLnsentinbxDesx) { - this.melpLnsentinbxDesx = melpLnsentinbxDesx; - } - - public String getMELP_LNSENTINBX_YEAR() { - return melpLnsentinbxYear; - } - - public void setMELP_LNSENTINBX_YEAR(String melpLnsentinbxYear) { - this.melpLnsentinbxYear = melpLnsentinbxYear; - } - - public String getMELP_LNPROLYSCT_DESC() { - return melpLnprolysctDesc; - } - - public void setMELP_LNPROLYSCT_DESC(String melpLnprolysctDesc) { - this.melpLnprolysctDesc = melpLnprolysctDesc; - } - - public String getMELP_LNPROSUCC_DESC() { - return melpLnprosuccDesc; - } - - public void setMELP_LNPROSUCC_DESC(String melpLnprosuccDesc) { - this.melpLnprosuccDesc = melpLnprosuccDesc; - } - - public String getMELP_LNDSCT_CMP_DESC() { - return melpLndsctCmpDesc; - } - - public void setMELP_LNDSCT_CMP_DESC(String melpLndsctCmpDesc) { - this.melpLndsctCmpDesc = melpLndsctCmpDesc; - } - - public String getMELP_LNDSCT_YEAR() { - return melpLndsctYear; - } - - public void setMELP_LNDSCT_YEAR(String melpLndsctYear) { - this.melpLndsctYear = melpLndsctYear; - } - - public String getMELP_LNMATTED_DESC() { - return melpLnmattedDesc; - } - - public void setMELP_LNMATTED_DESC(String melpLnmattedDesc) { - this.melpLnmattedDesc = melpLnmattedDesc; - } - - public String getMELP_LNEXTNODST_DESC() { - return LnextnodstDesc; - } - - public void setMELP_LNEXTNODST_DESC(String LnextnodstDesc) { - this.LnextnodstDesc = LnextnodstDesc; - } - - public String getMELP_LNINTRMETS_DESC() { - return LnintrmetsDesc; - } - - public void setMELP_LNINTRMETS_DESC(String LnintrmetsDesc) { - this.LnintrmetsDesc = LnintrmetsDesc; - } - - public String getMELP_LNSIZE() { - return melpLnsize; - } - - public void setMELP_LNSIZE(String melpLnsize) { - this.melpLnsize = melpLnsize; - } - - public String getMELP_LNSIZE_UNK_DESC() { - return melpLnsizeUnkDesc; - } - - public void setMELP_LNSIZE_UNK_DESC(String melpLnsizeUnkDesc) { - this.melpLnsizeUnkDesc = melpLnsizeUnkDesc; - } - - public String getMELP_LNSLNLARG_SIZE() { - return melpLnslnlargSize; - } - - public void setMELP_LNSLNLARG_SIZE(String melpLnslnlargSize) { - this.melpLnslnlargSize = melpLnslnlargSize; - } - - public String getMELP_LNIHC_DESC() { - return melpLnihcDesc; - } - - public void setMELP_LNIHC_DESC(String melpLnihcDesc) { - this.melpLnihcDesc = melpLnihcDesc; - } - - public String getMELP_LNIMM_S100_DESC() { - return melpLnimmS100Desc; - } - - public void setMELP_LNIMM_S100_DESC(String melpLnimmS100Desc) { - this.melpLnimmS100Desc = melpLnimmS100Desc; - } - - public String getMELP_LNIMMHMB45_DESC() { - return melpLnimmhmb45Desc; - } - - public void setMELP_LNIMMHMB45_DESC(String melpLnimmhmb45Desc) { - this.melpLnimmhmb45Desc = melpLnimmhmb45Desc; - } - - public String getMELP_LNIMM_MELA_DESC() { - return melpLnimmMelaDesc; - } - - public void setMELP_LNIMM_MELA_DESC(String melpLnimmMelaDesc) { - this.melpLnimmMelaDesc = melpLnimmMelaDesc; - } - - public String getMELI_PTID() { - return meliPtid; - } - - public void setMELI_PTID(String meliPtid) { - this.meliPtid = meliPtid; - } - - public String getMELI_DMP_PATIENT_ID() { - return meliDmpPatientId; - } - - public void setMELI_DMP_PATIENT_ID(String meliDmpPatientId) { - this.meliDmpPatientId = meliDmpPatientId; - } - - public String getMELI_DMP_SAMPLE_ID() { - return meliDmpSampleId; - } - - public void setMELI_DMP_SAMPLE_ID(String meliDmpSampleId) { - this.meliDmpSampleId = meliDmpSampleId; - } - - public String getMELI_REPORT_YEAR() { - return meliReportYear; - } - - public void setMELI_REPORT_YEAR(String meliReportYear) { - this.meliReportYear = meliReportYear; - } - - public String getMELI_PROCEDURE_YEAR() { - return meliProcedureYear; - } - - public void setMELI_PROCEDURE_YEAR(String meliProcedureYear) { - this.meliProcedureYear = meliProcedureYear; - } - - public String getMELI_TUMOR_TYPE() { - return meliTumorType; - } - - public void setMELI_TUMOR_TYPE(String meliTumorType) { - this.meliTumorType = meliTumorType; - } - - public String getMELI_PRIMARY_SITE() { - return meliPrimarySite; - } - - public void setMELI_PRIMARY_SITE(String meliPrimarySite) { - this.meliPrimarySite = meliPrimarySite; - } - - public String getMELI_MET_SITE() { - return meliMetSite; - } - - public void setMELI_MET_SITE(String meliMetSite) { - this.meliMetSite = meliMetSite; - } - - public String getMELMS_PTID() { - return melmsPtid; - } - - public void setMELMS_PTID(String melmsPtid) { - this.melmsPtid = melmsPtid; - } - - public String getMELMS_SITE_TYPE_DESC() { - return melmsSiteTypeDesc; - } - - public void setMELMS_SITE_TYPE_DESC(String melmsSiteTypeDesc) { - this.melmsSiteTypeDesc = melmsSiteTypeDesc; - } - - public String getMELMS_SITE_DESC() { - return melmsSiteDesc; - } - - public void setMELMS_SITE_DESC(String melmsSiteDesc) { - this.melmsSiteDesc = melmsSiteDesc; - } - - public String getMELMS_SITE_YEAR() { - return melmsSiteYear; - } - - public void setMELMS_SITE_YEAR(String melmsSiteYear) { - this.melmsSiteYear = melmsSiteYear; - } - - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("SAMPLE_ID"); - fieldNames.add("MELSPC_STAGE_YEAR"); - fieldNames.add("MELSPC_STG_GRP_NAME"); - fieldNames.add("MELG_STS_DESC"); - fieldNames.add("MELG_STS_SRCE_DESC"); - fieldNames.add("MELG_ACTV_STS_DESC"); - fieldNames.add("MELG_DERMAGRPHX_DESC"); - fieldNames.add("MELG_PRES_STG_YEAR"); - fieldNames.add("MELG_FAMILY_HX_DESC"); - fieldNames.add("MELG_1ST_RECUR_YEAR"); - fieldNames.add("MELG_LOCAL_DESC"); - fieldNames.add("MELG_NODAL_DESC"); - fieldNames.add("MELG_INTRANSIT_DESC"); - fieldNames.add("MELG_SYS_DESC"); - fieldNames.add("MELG_RECUR_NDSZ_DESC"); - fieldNames.add("MELG_RECUR_NODAL_NO"); - fieldNames.add("MELG_LDH"); - fieldNames.add("MELG_LDH_YEAR"); - fieldNames.add("MELG_METS_DESC"); - fieldNames.add("MELG_ADJVNT_TX_DESC"); - fieldNames.add("MELG_SYS_TX_DESC"); - fieldNames.add("MELG_RAD_TX_DESC"); - fieldNames.add("MELG_SURG_DESC"); - fieldNames.add("MELG_TISSUE_BANK_AVAIL"); - fieldNames.add("MELP_PRIM_SEQ"); - fieldNames.add("MELP_DX_YEAR"); - fieldNames.add("MELP_MSK_REVIEW_DESC"); - fieldNames.add("MELP_THICKNESS_MM"); - fieldNames.add("MELP_CLARK_LVL_DESC"); - fieldNames.add("MELP_ULCERATION_DESC"); - fieldNames.add("MELP_SITE_DESC"); - fieldNames.add("MELP_SUB_SITE_DESC"); - fieldNames.add("MELP_TILS_DESC"); - fieldNames.add("MELP_REGRESSION_DESC"); - fieldNames.add("MELP_MARGINS_DESC"); - fieldNames.add("MELP_MITIDX_UNK_DESC"); - fieldNames.add("MELP_HIST_TYPE_DESC"); - fieldNames.add("MELP_SATELLITES_DESC"); - fieldNames.add("MELP_EXT_SLIDES_DESC"); - fieldNames.add("MELP_LNORG_DX_DESC"); - fieldNames.add("MELP_LNCLIN_STS_DESC"); - fieldNames.add("MELP_LNSENTINBX_DESC"); - fieldNames.add("MELP_LNSENTINBX_YEAR"); - fieldNames.add("MELP_LNPROLYSCT_DESC"); - fieldNames.add("MELP_LNPROSUCC_DESC"); - fieldNames.add("MELP_LNDSCT_CMP_DESC"); - fieldNames.add("MELP_LNDSCT_YEAR"); - fieldNames.add("MELP_LNMATTED_DESC"); - fieldNames.add("MELP_LNEXTNODST_DESC"); - fieldNames.add("MELP_LNINTRMETS_DESC"); - fieldNames.add("MELP_LNSIZE"); - fieldNames.add("MELP_LNSIZE_UNK_DESC"); - fieldNames.add("MELP_LNSLNLARG_SIZE"); - fieldNames.add("MELP_LNIHC_DESC"); - fieldNames.add("MELP_LNIMM_S100_DESC"); - fieldNames.add("MELP_LNIMMHMB45_DESC"); - fieldNames.add("MELP_LNIMM_MELA_DESC"); - fieldNames.add("MELI_REPORT_YEAR"); - fieldNames.add("MELI_PROCEDURE_YEAR"); - fieldNames.add("MELI_TUMOR_TYPE"); - fieldNames.add("MELI_PRIMARY_SITE"); - fieldNames.add("MELI_MET_SITE"); - fieldNames.add("MELMS_SITE_TYPE_DESC"); - fieldNames.add("MELMS_SITE_DESC"); - fieldNames.add("MELMS_SITE_YEAR"); - - return fieldNames; - } - - public List getAllVariables() { - List fieldNames = getFieldNames(); - fieldNames.add("MELSPC_PTID"); - fieldNames.add("MELP_PTID"); - fieldNames.add("MELG_PTID"); - fieldNames.add("MELI_PTID"); - fieldNames.add("MELMS_PTID"); - - return fieldNames; - } - - public void setAdditionalProperty(String name, Object value) { - this.additionalProperties.put(name, value); - } - - public Map getAdditionalProperties() { - return this.additionalProperties; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantNormalizedClinicalRecord.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantNormalizedClinicalRecord.java deleted file mode 100644 index ae2433463..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantNormalizedClinicalRecord.java +++ /dev/null @@ -1,859 +0,0 @@ -/* - * Copyright (c) 2018, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -public class Skcm_mskcc_2015_chantNormalizedClinicalRecord { - private String patientId; - private String sampleId; - private String stageYear; - private String stgGrpName; - private String vitalStatus; - private String stsSrcDesc; - private String lastStatus; - private String dermagrphxDes; - private String presStgYear; - private String familyHistory; - private String timeToFirstRecurrence; - private String localDesc; - private String nodalDesc; - private String intransitDesc; - private String sysDesc; - private String recurNdszDes; - private String recurNodalNo; - private String ldh; - private String ldhYear; - private String metastasis; - private String adjvntTx; - private String systemicTreatment; - private String treatmentRadiation; - private String surgery; - private String tissueBankAvail; - private String primSeq; - private String yearOfDiagnosis; - private String mskReviewDes; - private String tumorThicknessMeasurement; - private String clarkLevelAtDiagnosis; - private String primaryMelanomaTumorUlceration; - private String tumorTissueSite; - private String detailedPrimarySite; - private String lymphocyteInfiltration; - private String regressionDes; - private String marginStatus; - private String mitoticIndex; - private String histologicalType; - private String satellitesDes; - private String extSlidesDes; - private String primaryLymphNodePresentationAssessment; - private String lnclinStsDes; - private String lnsentinbxDes; - private String lnsentinbxYea; - private String lnprolysctDes; - private String lnprosuccDesc; - private String lndsctCmpDes; - private String lndsctYear; - private String lnmattedDesc; - private String lnextnodstDes; - private String lnintrmetsDes; - private String lnsize; - private String lnsizeUnkDes; - private String lnslnlargSize; - private String lnihcDesc; - private String s100Stain; - private String lnimmhmb45Des; - private String lnimmMelaDes; - private String reportYear; - private String procedureYear; - private String tumorType; - private String primarySite; - private String metastaticSite; - private String initialMetDisease; - private String otherSitesOfMets; - private String yearMetDiseaseIdentified; - - private Map additionalProperties; - - public Skcm_mskcc_2015_chantNormalizedClinicalRecord() { - } - - public Skcm_mskcc_2015_chantNormalizedClinicalRecord( - String patientId, - String sampleId, - String stageYear, - String stgGrpName, - String vitalStatus, - String stsSrcDesc, - String lastStatus, - String dermagrphxDes, - String presStgYear, - String familyHistory, - String timeToFirstRecurrence, - String localDesc, - String nodalDesc, - String intransitDesc, - String sysDesc, - String recurNdszDes, - String recurNodalNo, - String ldh, - String ldhYear, - String metastasis, - String adjvntTx, - String systemicTreatment, - String treatmentRadiation, - String surgery, - String tissueBankAvail, - String primSeq, - String yearOfDiagnosis, - String mskReviewDes, - String tumorThicknessMeasurement, - String clarkLevelAtDiagnosis, - String primaryMelanomaTumorUlceration, - String tumorTissueSite, - String detailedPrimarySite, - String lymphocyteInfiltration, - String regressionDes, - String marginStatus, - String mitoticIndex, - String histologicalType, - String satellitesDes, - String extSlidesDes, - String primaryLymphNodePresentationAssessment, - String lnclinStsDes, - String lnsentinbxDes, - String lnsentinbxYea, - String lnprolysctDes, - String lnprosuccDesc, - String lndsctCmpDes, - String lndsctYear, - String lnmattedDesc, - String lnextnodstDes, - String lnintrmetsDes, - String lnsize, - String lnsizeUnkDes, - String lnslnlargSize, - String lnihcDesc, - String s100Stain, - String lnimmhmb45Des, - String lnimmMelaDes, - String reportYear, - String procedureYear, - String tumorType, - String primarySite, - String metastaticSite, - String initialMetDisease, - String otherSitesOfMets, - String yearMetDiseaseIdentified) { - this.patientId = ClinicalValueUtil.defaultWithNA(patientId); - this.sampleId = ClinicalValueUtil.defaultWithNA(sampleId); - this.stageYear = ClinicalValueUtil.defaultWithNA(stageYear); - this.stgGrpName = ClinicalValueUtil.defaultWithNA(stgGrpName); - this.vitalStatus = ClinicalValueUtil.defaultWithNA(vitalStatus); - this.stsSrcDesc = ClinicalValueUtil.defaultWithNA(stsSrcDesc); - this.lastStatus = ClinicalValueUtil.defaultWithNA(lastStatus); - this.dermagrphxDes = ClinicalValueUtil.defaultWithNA(dermagrphxDes); - this.presStgYear = ClinicalValueUtil.defaultWithNA(presStgYear); - this.familyHistory = ClinicalValueUtil.defaultWithNA(familyHistory); - this.timeToFirstRecurrence = ClinicalValueUtil.defaultWithNA(timeToFirstRecurrence); - this.localDesc = ClinicalValueUtil.defaultWithNA(localDesc); - this.nodalDesc = ClinicalValueUtil.defaultWithNA(nodalDesc); - this.intransitDesc = ClinicalValueUtil.defaultWithNA(intransitDesc); - this.sysDesc = ClinicalValueUtil.defaultWithNA(sysDesc); - this.recurNdszDes = ClinicalValueUtil.defaultWithNA(recurNdszDes); - this.recurNodalNo = ClinicalValueUtil.defaultWithNA(recurNodalNo); - this.ldh = ClinicalValueUtil.defaultWithNA(ldh); - this.ldhYear = ClinicalValueUtil.defaultWithNA(ldhYear); - this.metastasis = ClinicalValueUtil.defaultWithNA(metastasis); - this.adjvntTx = ClinicalValueUtil.defaultWithNA(adjvntTx); - this.systemicTreatment = ClinicalValueUtil.defaultWithNA(systemicTreatment); - this.treatmentRadiation = ClinicalValueUtil.defaultWithNA(treatmentRadiation); - this.surgery = ClinicalValueUtil.defaultWithNA(surgery); - this.tissueBankAvail = ClinicalValueUtil.defaultWithNA(tissueBankAvail); - this.primSeq = ClinicalValueUtil.defaultWithNA(primSeq); - this.yearOfDiagnosis = ClinicalValueUtil.defaultWithNA(yearOfDiagnosis); - this.mskReviewDes = ClinicalValueUtil.defaultWithNA(mskReviewDes); - this.tumorThicknessMeasurement = ClinicalValueUtil.defaultWithNA(tumorThicknessMeasurement); - this.clarkLevelAtDiagnosis = ClinicalValueUtil.defaultWithNA(clarkLevelAtDiagnosis); - this.primaryMelanomaTumorUlceration = ClinicalValueUtil.defaultWithNA(primaryMelanomaTumorUlceration); - this.tumorTissueSite = ClinicalValueUtil.defaultWithNA(tumorTissueSite); - this.detailedPrimarySite = ClinicalValueUtil.defaultWithNA(detailedPrimarySite); - this.lymphocyteInfiltration = ClinicalValueUtil.defaultWithNA(lymphocyteInfiltration); - this.regressionDes = ClinicalValueUtil.defaultWithNA(regressionDes); - this.marginStatus = ClinicalValueUtil.defaultWithNA(marginStatus); - this.mitoticIndex = ClinicalValueUtil.defaultWithNA(mitoticIndex); - this.histologicalType = ClinicalValueUtil.defaultWithNA(histologicalType); - this.satellitesDes = ClinicalValueUtil.defaultWithNA(satellitesDes); - this.extSlidesDes = ClinicalValueUtil.defaultWithNA(extSlidesDes); - this.primaryLymphNodePresentationAssessment = ClinicalValueUtil.defaultWithNA(primaryLymphNodePresentationAssessment); - this.lnclinStsDes = ClinicalValueUtil.defaultWithNA(lnclinStsDes); - this.lnsentinbxDes = ClinicalValueUtil.defaultWithNA(lnsentinbxDes); - this.lnsentinbxYea = ClinicalValueUtil.defaultWithNA(lnsentinbxYea); - this.lnprolysctDes = ClinicalValueUtil.defaultWithNA(lnprolysctDes); - this.lnprosuccDesc = ClinicalValueUtil.defaultWithNA(lnprosuccDesc); - this.lndsctCmpDes = ClinicalValueUtil.defaultWithNA(lndsctCmpDes); - this.lndsctYear = ClinicalValueUtil.defaultWithNA(lndsctYear); - this.lnmattedDesc = ClinicalValueUtil.defaultWithNA(lnmattedDesc); - this.lnextnodstDes = ClinicalValueUtil.defaultWithNA(lnextnodstDes); - this.lnintrmetsDes = ClinicalValueUtil.defaultWithNA(lnintrmetsDes); - this.lnsize = ClinicalValueUtil.defaultWithNA(lnsize); - this.lnsizeUnkDes = ClinicalValueUtil.defaultWithNA(lnsizeUnkDes); - this.lnslnlargSize = ClinicalValueUtil.defaultWithNA(lnslnlargSize); - this.lnihcDesc = ClinicalValueUtil.defaultWithNA(lnihcDesc); - this.s100Stain = ClinicalValueUtil.defaultWithNA(s100Stain); - this.lnimmhmb45Des = ClinicalValueUtil.defaultWithNA(lnimmhmb45Des); - this.lnimmMelaDes = ClinicalValueUtil.defaultWithNA(lnimmMelaDes); - this.reportYear = ClinicalValueUtil.defaultWithNA(reportYear); - this.procedureYear = ClinicalValueUtil.defaultWithNA(procedureYear); - this.tumorType = ClinicalValueUtil.defaultWithNA(tumorType); - this.primarySite = ClinicalValueUtil.defaultWithNA(primarySite); - this.metastaticSite = ClinicalValueUtil.defaultWithNA(metastaticSite); - this.initialMetDisease = ClinicalValueUtil.defaultWithNA(initialMetDisease); - this.otherSitesOfMets = ClinicalValueUtil.defaultWithNA(otherSitesOfMets); - this.yearMetDiseaseIdentified = ClinicalValueUtil.defaultWithNA(yearMetDiseaseIdentified); - } - - public String getPATIENT_ID() { - return patientId; - } - - public void setPATIENT_ID(String patientId) { - this.patientId = patientId; - } - - public String getSAMPLE_ID() { - return sampleId; - } - - public void setSAMPLE_ID(String sampleId) { - this.sampleId = sampleId; - } - - public String getSTAGE_YEAR() { - return this.stageYear; - } - - public void setSTAGE_YEAR(String stageYear) { - this.stageYear = stageYear; - } - - public String getSTG_GRP_NAME() { - return this.stgGrpName; - } - - public void setSTG_GRP_NAME(String stgGrpName) { - this.stgGrpName = stgGrpName; - } - - public String getVITAL_STATUS() { - return this.vitalStatus; - } - - public void setVITAL_STATUS(String vitalStatus) { - this.vitalStatus = vitalStatus; - } - - public String getSTS_SRCE_DESC() { - return this.stsSrcDesc; - } - - public void setSTS_SRCE_DESC(String stsSrcDesc) { - this.stsSrcDesc = stsSrcDesc; - } - - public String getLAST_STATUS() { - return this.lastStatus; - } - - public void setLAST_STATUS(String lastStatus) { - this.lastStatus = lastStatus; - } - - public String getDERMAGRPHX_DES() { - return this.dermagrphxDes; - } - - public void setDERMAGRPHX_DES(String dermagrphxDes) { - this.dermagrphxDes = dermagrphxDes; - } - - public String getPRES_STG_YEAR() { - return this.presStgYear; - } - - public void setPRES_STG_YEAR(String presStgYear) { - this.presStgYear = presStgYear; - } - - public String getFAMILY_HISTORY() { - return this.familyHistory; - } - - public void setFAMILY_HISTORY(String familyHistory) { - this.familyHistory = familyHistory; - } - - public String getTIME_TO_FIRST_RECURRENCE() { - return this.timeToFirstRecurrence; - } - - public void setTIME_TO_FIRST_RECURRENCE(String timeToFirstRecurrence) { - this.timeToFirstRecurrence = timeToFirstRecurrence; - } - - public String getLOCAL_DESC() { - return this.localDesc; - } - - public void setLOCAL_DESC(String localDesc) { - this.localDesc = localDesc; - } - - public String getNODAL_DESC() { - return this.nodalDesc; - } - - public void setNODAL_DESC(String nodalDesc) { - this.nodalDesc = nodalDesc; - } - - public String getINTRANSIT_DESC() { - return this.intransitDesc; - } - - public void setINTRANSIT_DESC(String intransitDesc) { - this.intransitDesc = intransitDesc; - } - - public String getSYS_DESC() { - return this.sysDesc; - } - - public void setSYS_DESC(String sysDesc) { - this.sysDesc = sysDesc; - } - - public String getRECUR_NDSZ_DES() { - return this.recurNdszDes; - } - - public void setRECUR_NDSZ_DES(String recurNdszDes) { - this.recurNdszDes = recurNdszDes; - } - - public String getRECUR_NODAL_NO() { - return this.recurNodalNo; - } - - public void setRECUR_NODAL_NO(String recurNodalNo) { - this.recurNodalNo = recurNodalNo; - } - - public String getLDH_LEVEL() { - return this.ldh; - } - - public void setLDH_LEVEL(String ldh) { - this.ldh = ldh; - } - - public String getLDH_YEAR() { - return this.ldhYear; - } - - public void setLDH_YEAR(String ldhYear) { - this.ldhYear = ldhYear; - } - - public String getMETASTASIS() { - return this.metastasis; - } - - public void setMETASTASIS(String metastasis) { - this.metastasis = metastasis; - } - - public String getADJUVANT_TX() { - return this.adjvntTx; - } - - public void setADJUVANT_TX(String adjvntTx) { - this.adjvntTx = adjvntTx; - } - - public String getSYSTEMIC_TREATMENT() { - return this.systemicTreatment; - } - - public void setSYSTEMIC_TREATMENT(String systemicTreatment) { - this.systemicTreatment = systemicTreatment; - } - - public String getTREATMENT_RADIATION() { - return this.treatmentRadiation; - } - - public void setTREATMENT_RADIATION(String treatmentRadiation) { - this.treatmentRadiation = treatmentRadiation; - } - - public String getSURGERY() { - return this.surgery; - } - - public void setSURGERY(String surgery) { - this.surgery = surgery; - } - - public String getTISSUE_BANK_AVAIL() { - return this.tissueBankAvail; - } - - public void setTISSUE_BANK_AVAIL(String tissueBankAvail) { - this.tissueBankAvail = tissueBankAvail; - } - - public String getPRIM_SEQ() { - return this.primSeq; - } - - public void setPRIM_SEQ(String primSeq) { - this.primSeq = primSeq; - } - - public String getYEAR_OF_DIAGNOSIS() { - return this.yearOfDiagnosis; - } - - public void setYEAR_OF_DIAGNOSIS(String yearOfDiagnosis) { - this.yearOfDiagnosis = yearOfDiagnosis; - } - - public String getMSK_REVIEW_DES() { - return this.mskReviewDes; - } - - public void setMSK_REVIEW_DES(String mskReviewDes) { - this.mskReviewDes = mskReviewDes; - } - - public String getTUMOR_THICKNESS_MEASUREMENT() { - return this.tumorThicknessMeasurement; - } - - public void setTUMOR_THICKNESS_MEASUREMENT(String tumorThicknessMeasurement) { - this.tumorThicknessMeasurement = tumorThicknessMeasurement; - } - - public String getCLARK_LEVEL_AT_DIAGNOSIS() { - return this.clarkLevelAtDiagnosis; - } - - public void setCLARK_LEVEL_AT_DIAGNOSIS(String clarkLevelAtDiagnosis) { - this.clarkLevelAtDiagnosis = clarkLevelAtDiagnosis; - } - - public String getPRIMARY_MELANOMA_TUMOR_ULCERATION() { - return this.primaryMelanomaTumorUlceration; - } - - public void setPRIMARY_MELANOMA_TUMOR_ULCERATION(String primaryMelanomaTumorUlceration) { - this.primaryMelanomaTumorUlceration = primaryMelanomaTumorUlceration; - } - - public String getTUMOR_TISSUE_SITE() { - return this.tumorTissueSite; - } - - public void setTUMOR_TISSUE_SITE(String tumorTissueSite) { - this.tumorTissueSite = tumorTissueSite; - } - - public String getDETAILED_PRIMARY_SITE() { - return this.detailedPrimarySite; - } - - public void setDETAILED_PRIMARY_SITE(String detailedPrimarySite) { - this.detailedPrimarySite = detailedPrimarySite; - } - - public String getLYMPHOCYTE_INFILTRATION() { - return this.lymphocyteInfiltration; - } - - public void setLYMPHOCYTE_INFILTRATION(String lymphocyteInfiltration) { - this.lymphocyteInfiltration = lymphocyteInfiltration; - } - - public String getREGRESSION_DES() { - return this.regressionDes; - } - - public void setREGRESSION_DES(String regressionDes) { - this.regressionDes = regressionDes; - } - - public String getMARGIN_STATUS() { - return this.marginStatus; - } - - public void setMARGIN_STATUS(String marginStatus) { - this.marginStatus = marginStatus; - } - - public String getMITOTIC_INDEX() { - return this.mitoticIndex; - } - - public void setMITOTIC_INDEX(String mitoticIndex) { - this.mitoticIndex = mitoticIndex; - } - - public String getHISTOLOGICAL_TYPE() { - return this.histologicalType; - } - - public void setHISTOLOGICAL_TYPE(String histologicalType) { - this.histologicalType = histologicalType; - } - - public String getSATELLITES_DES() { - return this.satellitesDes; - } - - public void setSATELLITES_DES(String satellitesDes) { - this.satellitesDes = satellitesDes; - } - - public String getEXT_SLIDES_DES() { - return this.extSlidesDes; - } - - public void setEXT_SLIDES_DES(String extSlidesDes) { - this.extSlidesDes = extSlidesDes; - } - - public String getPRIMARY_LYMPH_NODE_PRESENTATION_ASSESSMENT() { - return this.primaryLymphNodePresentationAssessment; - } - - public void setPRIMARY_LYMPH_NODE_PRESENTATION_ASSESSMENT(String primaryLymphNodePresentationAssessment) { - this.primaryLymphNodePresentationAssessment = primaryLymphNodePresentationAssessment; - } - - public String getLNCLIN_STS_DES() { - return this.lnclinStsDes; - } - - public void setLNCLIN_STS_DES(String lnclinStsDes) { - this.lnclinStsDes = lnclinStsDes; - } - - public String getLNSENTINBX_DES() { - return this.lnsentinbxDes; - } - - public void setLNSENTINBX_DES(String lnsentinbxDes) { - this.lnsentinbxDes = lnsentinbxDes; - } - - public String getLNSENTINBX_YEA() { - return this.lnsentinbxYea; - } - - public void setLNSENTINBX_YEA(String lnsentinbxYea) { - this.lnsentinbxYea = lnsentinbxYea; - } - - public String getLNPROLYSCT_DES() { - return this.lnprolysctDes; - } - - public void setLNPROLYSCT_DES(String lnprolysctDes) { - this.lnprolysctDes = lnprolysctDes; - } - - public String getLNPROSUCC_DESC() { - return this.lnprosuccDesc; - } - - public void setLNPROSUCC_DESC(String lnprosuccDesc) { - this.lnprosuccDesc = lnprosuccDesc; - } - - public String getLNDSCT_CMP_DES() { - return this.lndsctCmpDes; - } - - public void setLNDSCT_CMP_DES(String lndsctCmpDes) { - this.lndsctCmpDes = lndsctCmpDes; - } - - public String getLNDSCT_YEAR() { - return this.lndsctYear; - } - - public void setLNDSCT_YEAR(String lndsctYear) { - this.lndsctYear = lndsctYear; - } - - public String getLNMATTED_DESC() { - return this.lnmattedDesc; - } - - public void setLNMATTED_DESC(String lnmattedDesc) { - this.lnmattedDesc = lnmattedDesc; - } - - public String getLNEXTNODST_DES() { - return this.lnextnodstDes; - } - - public void setLNEXTNODST_DES(String lnextnodstDes) { - this.lnextnodstDes = lnextnodstDes; - } - - public String getLNINTRMETS_DES() { - return this.lnintrmetsDes; - } - - public void setLNINTRMETS_DES(String lnintrmetsDes) { - this.lnintrmetsDes = lnintrmetsDes; - } - - public String getLNSIZE() { - return this.lnsize; - } - - public void setLNSIZE(String lnsize) { - this.lnsize = lnsize; - } - - public String getLNSIZE_UNK_DES() { - return this.lnsizeUnkDes; - } - - public void setLNSIZE_UNK_DES(String lnsizeUnkDes) { - this.lnsizeUnkDes = lnsizeUnkDes; - } - - public String getLNSLNLARG_SIZE() { - return this.lnslnlargSize; - } - - public void setLNSLNLARG_SIZE(String lnslnlargSize) { - this.lnslnlargSize = lnslnlargSize; - } - - public String getLNIHC_DESC() { - return this.lnihcDesc; - } - - public void setLNIHC_DESC(String lnihcDesc) { - this.lnihcDesc = lnihcDesc; - } - - public String getS_100_STAIN() { - return this.s100Stain; - } - - public void setS_100_STAIN(String s100Stain) { - this.s100Stain = s100Stain; - } - - public String getLNIMMHMB45_DES() { - return this.lnimmhmb45Des; - } - - public void setLNIMMHMB45_DES(String lnimmhmb45Des) { - this.lnimmhmb45Des = lnimmhmb45Des; - } - - public String getLNIMM_MELA_DES() { - return this.lnimmMelaDes; - } - - public void setLNIMM_MELA_DES(String lnimmMelaDes) { - this.lnimmMelaDes = lnimmMelaDes; - } - - public String getREPORT_YEAR() { - return this.reportYear; - } - - public void setREPORT_YEAR(String reportYear) { - this.reportYear = reportYear; - } - - public String getPROCEDURE_YEAR() { - return this.procedureYear; - } - - public void setPROCEDURE_YEAR(String procedureYear) { - this.procedureYear = procedureYear; - } - - public String getTUMOR_TYPE() { - return this.tumorType; - } - - public void setTUMOR_TYPE(String tumorType) { - this.tumorType = tumorType; - } - - public String getPRIMARY_SITE() { - return this.primarySite; - } - - public void setPRIMARY_SITE(String primarySite) { - this.primarySite = primarySite; - } - - public String getMETASTATIC_SITE() { - return this.metastaticSite; - } - - public void setMETASTATIC_SITE(String metastaticSite) { - this.metastaticSite = metastaticSite; - } - - public String getINITIAL_MET_DISEASE() { - return this.initialMetDisease; - } - - public void setINITIAL_MET_DISEASE(String initialMetDisease) { - this.initialMetDisease = initialMetDisease; - } - - public String getOTHER_SITES_OF_METS() { - return this.otherSitesOfMets; - } - - public void setOTHER_SITES_OF_METS(String otherSitesOfMets) { - this.otherSitesOfMets = otherSitesOfMets; - } - - public String getYEAR_MET_DISEASE_IDENTIFIED() { - return this.yearMetDiseaseIdentified; - } - - public void setYEAR_MET_DISEASE_IDENTIFIED(String yearMetDiseaseIdentified) { - this.yearMetDiseaseIdentified = yearMetDiseaseIdentified; - } - - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - fieldNames.add("PATIENT_ID"); - fieldNames.add("SAMPLE_ID"); - fieldNames.add("STAGE_YEAR"); - fieldNames.add("STG_GRP_NAME"); - fieldNames.add("VITAL_STATUS"); - fieldNames.add("STS_SRCE_DESC"); - fieldNames.add("LAST_STATUS"); - fieldNames.add("DERMAGRPHX_DES"); - fieldNames.add("PRES_STG_YEAR"); - fieldNames.add("FAMILY_HISTORY"); - fieldNames.add("TIME_TO_FIRST_RECURRENCE"); - fieldNames.add("LOCAL_DESC"); - fieldNames.add("NODAL_DESC"); - fieldNames.add("INTRANSIT_DESC"); - fieldNames.add("SYS_DESC"); - fieldNames.add("RECUR_NDSZ_DES"); - fieldNames.add("RECUR_NODAL_NO"); - fieldNames.add("LDH_LEVEL"); - fieldNames.add("LDH_YEAR"); - fieldNames.add("METASTASIS"); - fieldNames.add("ADJUVANT_TX"); - fieldNames.add("SYSTEMIC_TREATMENT"); - fieldNames.add("TREATMENT_RADIATION"); - fieldNames.add("SURGERY"); - fieldNames.add("TISSUE_BANK_AVAIL"); - fieldNames.add("PRIM_SEQ"); - fieldNames.add("YEAR_OF_DIAGNOSIS"); - fieldNames.add("MSK_REVIEW_DES"); - fieldNames.add("TUMOR_THICKNESS_MEASUREMENT"); - fieldNames.add("CLARK_LEVEL_AT_DIAGNOSIS"); - fieldNames.add("PRIMARY_MELANOMA_TUMOR_ULCERATION"); - fieldNames.add("TUMOR_TISSUE_SITE"); - fieldNames.add("DETAILED_PRIMARY_SITE"); - fieldNames.add("LYMPHOCYTE_INFILTRATION"); - fieldNames.add("REGRESSION_DES"); - fieldNames.add("MARGIN_STATUS"); - fieldNames.add("MITOTIC_INDEX"); - fieldNames.add("HISTOLOGICAL_TYPE"); - fieldNames.add("SATELLITES_DES"); - fieldNames.add("EXT_SLIDES_DES"); - fieldNames.add("PRIMARY_LYMPH_NODE_PRESENTATION_ASSESSMENT"); - fieldNames.add("LNCLIN_STS_DES"); - fieldNames.add("LNSENTINBX_DES"); - fieldNames.add("LNSENTINBX_YEA"); - fieldNames.add("LNPROLYSCT_DES"); - fieldNames.add("LNPROSUCC_DESC"); - fieldNames.add("LNDSCT_CMP_DES"); - fieldNames.add("LNDSCT_YEAR"); - fieldNames.add("LNMATTED_DESC"); - fieldNames.add("LNEXTNODST_DES"); - fieldNames.add("LNINTRMETS_DES"); - fieldNames.add("LNSIZE"); - fieldNames.add("LNSIZE_UNK_DES"); - fieldNames.add("LNSLNLARG_SIZE"); - fieldNames.add("LNIHC_DESC"); - fieldNames.add("S_100_STAIN"); - fieldNames.add("LNIMMHMB45_DES"); - fieldNames.add("LNIMM_MELA_DES"); - fieldNames.add("REPORT_YEAR"); - fieldNames.add("PROCEDURE_YEAR"); - fieldNames.add("TUMOR_TYPE"); - fieldNames.add("PRIMARY_SITE"); - fieldNames.add("METASTATIC_SITE"); - fieldNames.add("INITIAL_MET_DISEASE"); - fieldNames.add("OTHER_SITES_OF_METS"); - fieldNames.add("YEAR_MET_DISEASE_IDENTIFIED"); - return fieldNames; - } - - public void setAdditionalProperty(String name, Object value) { - this.additionalProperties.put(name, value); - } - - public Map getAdditionalProperties() { - return this.additionalProperties; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineAdjuvantTx.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineAdjuvantTx.java deleted file mode 100644 index bc9b42e14..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineAdjuvantTx.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.ArrayList; -import java.util.List; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantTimelineAdjuvantTx implements Skcm_mskcc_2015_chantTimelineRecord { - - private String melatPtid; - private Integer melatAdjtxTypCd; - private String melatAdjtxTypDesc; - private String melatAdjtxTypeOth; - private Integer melatAdjtxStrtYear; - private Integer melatAdjtxEndYear; - private Integer melatAdjtxDaysDuration; - - public Skcm_mskcc_2015_chantTimelineAdjuvantTx() {} - - public Skcm_mskcc_2015_chantTimelineAdjuvantTx(String melatPtid, - Integer melatAdjtxTypCd, - String melatAdjtxTypDesc, - String melatAdjtxTypeOth, - Integer melatAdjtxStrtYear, - Integer melatAdjtxEndYear, - Integer melatAdjtxDaysDuration) { - this.melatPtid = ClinicalValueUtil.defaultWithNA(melatPtid); - this.melatAdjtxTypCd = melatAdjtxTypCd != null ? melatAdjtxTypCd : -1; - this.melatAdjtxTypDesc = ClinicalValueUtil.defaultWithNA(melatAdjtxTypDesc); - this.melatAdjtxTypeOth = ClinicalValueUtil.defaultWithNA(melatAdjtxTypeOth); - this.melatAdjtxStrtYear = melatAdjtxStrtYear != null ? melatAdjtxStrtYear : -1; - this.melatAdjtxEndYear = melatAdjtxEndYear != null ? melatAdjtxEndYear : -1; - this.melatAdjtxDaysDuration = melatAdjtxDaysDuration != null ? melatAdjtxDaysDuration : -1; - } - - public String getMELAT_PTID() { - return melatPtid; - } - - public void setMELAT_PTID(String melatPtid) { - this.melatPtid = melatPtid; - } - - public Integer getMELAT_ADJTX_TYP_CD() { - return melatAdjtxTypCd; - } - - public void setMELAT_ADJTX_TYP_CD(Integer melatAdjtxTypCd) { - this.melatAdjtxTypCd = melatAdjtxTypCd; - } - - public String getMELAT_ADJTX_TYP_DESC() { - return melatAdjtxTypDesc; - } - - public void setMELAT_ADJTX_TYP_DESC(String melatAdjtxTypDesc) { - this.melatAdjtxTypDesc = melatAdjtxTypDesc; - } - - public String getMELAT_ADJTX_TYPE_OTH() { - return melatAdjtxTypeOth; - } - - public void setMELAT_ADJTX_TYPE_OTH(String melatAdjtxTypeOth) { - this.melatAdjtxTypeOth = melatAdjtxTypeOth; - } - - public Integer getMELAT_ADJTX_STRT_YEAR() { - return melatAdjtxStrtYear; - } - - public void setMELAT_ADJTX_STRT_YEAR(Integer melatAdjtxStrtYear) { - this.melatAdjtxStrtYear = melatAdjtxStrtYear; - } - - public Integer getMELAT_ADJTX_END_YEAR() { - return melatAdjtxEndYear; - } - - public void setMELAT_ADJTX_END_YEAR(Integer melatAdjtxEndYear) { - this.melatAdjtxEndYear = melatAdjtxEndYear; - } - - public Integer getMELAT_ADJTX_DAYS_DURATION() { - return melatAdjtxDaysDuration; - } - - public void setMELAT_ADJTX_DAYS_DURATION(Integer melatAdjtxDaysDuration) { - this.melatAdjtxDaysDuration = melatAdjtxDaysDuration; - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("STOP_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("TREATMENT_TYPE"); - fieldNames.add("SUBTYPE"); - - return fieldNames; - } - - @Override - public String getPATIENT_ID() { - return melatPtid; - } - - // due to insufficient data in darwin, start date is 0 - @Override - public String getSTART_DATE() { - return "0"; - } - - @Override - public String getSTOP_DATE() { - return String.valueOf(melatAdjtxDaysDuration); - } - - @Override - public String getEVENT_TYPE() { - return "TREATMENT"; - } - - @Override - public String getTREATMENT_TYPE() { - return melatAdjtxTypDesc; - - } - - @Override - public String getSUBTYPE() { - return melatAdjtxTypeOth; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRadTherapy.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRadTherapy.java deleted file mode 100644 index e1ba14da5..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRadTherapy.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.ArrayList; -import java.util.List; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantTimelineRadTherapy implements Skcm_mskcc_2015_chantTimelineRecord { - - private String melrtPtid; - private String melrtRttxTypeDesc; - private String melrtRttxAdjDesc; - private Integer melrtRttxStrtYear; - private Integer melrtRttxEndDt; - private Integer melatRttxDaysDuration; - - public Skcm_mskcc_2015_chantTimelineRadTherapy() {} - - public Skcm_mskcc_2015_chantTimelineRadTherapy(String melrtPtid, - String melrtRttxTypeDesc, - String melrtRttxAdjDesc, - Integer melrtRttxStrtYear, - Integer melrtRttxEndDt, - Integer melatRttxDaysDuration - ) { - this.melrtPtid = ClinicalValueUtil.defaultWithNA(melrtPtid); - this.melrtRttxTypeDesc = ClinicalValueUtil.defaultWithNA(melrtRttxTypeDesc); - this.melrtRttxAdjDesc = ClinicalValueUtil.defaultWithNA(melrtRttxAdjDesc); - this.melrtRttxStrtYear = melrtRttxStrtYear != null ? melrtRttxStrtYear : -1; - this.melrtRttxEndDt = melrtRttxEndDt != null ? melrtRttxEndDt : -1; - this.melatRttxDaysDuration = melatRttxDaysDuration != null ? melatRttxDaysDuration : -1; - } - - public String getMELRT_PTID() { - return melrtPtid; - } - - public void setMELRT_PTID(String melrtPtid) { - this.melrtPtid = melrtPtid; - } - - public String getMELRT_RTTX_TYPE_DESC() { - return melrtRttxTypeDesc; - } - - public void setMELRT_RTTX_TYPE_DESC(String melrtRttxTypeDesc) { - this.melrtRttxTypeDesc = melrtRttxTypeDesc; - } - - public String getMELRT_RTTX_ADJ_DESC() { - return melrtRttxAdjDesc; - } - - public void setMELRT_RTTX_ADJ_DESC(String melrtRttxAdjDesc) { - this.melrtRttxAdjDesc = melrtRttxAdjDesc; - } - - public Integer getMELRT_RTTX_STRT_YEAR() { - return melrtRttxStrtYear; - } - - public void setMELRT_RTTX_STRT_YEAR(Integer melrtRttxStrtYear) { - this.melrtRttxStrtYear = melrtRttxStrtYear; - } - - public Integer getMELRT_RTTX_END_DT() { - return melrtRttxEndDt; - } - - public void setMELRT_RTTX_END_DT(Integer melrtRttxEndDt) { - this.melrtRttxEndDt = melrtRttxEndDt; - } - - public Integer getMELAT_RTTX_DAYS_DURATION() { - return melatRttxDaysDuration; - } - - public void setMELAT_RTTX_DAYS_DURATION(Integer melatRttxDaysDuration) { - this.melatRttxDaysDuration = melatRttxDaysDuration; - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("STOP_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("TREATMENT_TYPE"); - fieldNames.add("SUBTYPE"); - - return fieldNames; - } - - @Override - public String getPATIENT_ID() { - return melrtPtid; - } - - // due to insufficient data in darwin, start date is 0 - @Override - public String getSTART_DATE() { - return "0"; - } - - @Override - public String getSTOP_DATE() { - return String.valueOf(melatRttxDaysDuration); - } - - @Override - public String getEVENT_TYPE() { - return "TREATMENT"; - } - - @Override - public String getTREATMENT_TYPE() { - return melrtRttxTypeDesc; - - } - - @Override - public String getSUBTYPE() { - return melrtRttxAdjDesc; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRecord.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRecord.java deleted file mode 100644 index 90c1a710d..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineRecord.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.ArrayList; -import java.util.List; - -/** - * - * @author heinsz - */ -public interface Skcm_mskcc_2015_chantTimelineRecord { - - String getPATIENT_ID(); - String getSTART_DATE(); - String getSTOP_DATE(); - String getEVENT_TYPE(); - String getTREATMENT_TYPE(); - String getSUBTYPE(); - List getFieldNames(); -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineSystemicTx.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineSystemicTx.java deleted file mode 100644 index 4bb6219c0..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/model/Skcm_mskcc_2015_chantTimelineSystemicTx.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.model; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.cbioportal.cmo.pipelines.common.util.ClinicalValueUtil; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantTimelineSystemicTx implements Skcm_mskcc_2015_chantTimelineRecord { - - private String melstPtid; - private String melstSystxTypDesc; - private String melstSystxTypeOth; - private Integer melstSystxStrtYear; - private Integer melstSystxEndYear; - private Integer melstSystxDaysDuration; - private List EXCLUDED_TREATMENTS = Arrays.asList("Vaccine, please specify", "CombChemo, please specify", "Other, please specify", "HILP/ILI, please specify"); - - public Skcm_mskcc_2015_chantTimelineSystemicTx() {} - - public Skcm_mskcc_2015_chantTimelineSystemicTx(String melstPtid, - String melstSystxTypDesc, - String melstSystxTypeOth, - Integer melstSystxStrtYear, - Integer melstSystxEndYear, - Integer melstSystxDaysDuration - ) { - this.melstPtid = ClinicalValueUtil.defaultWithNA(melstPtid); - this.melstSystxTypDesc = ClinicalValueUtil.defaultWithNA(melstSystxTypDesc); - this.melstSystxTypeOth = ClinicalValueUtil.defaultWithNA(melstSystxTypeOth); - this.melstSystxStrtYear = melstSystxStrtYear != null ? melstSystxStrtYear : -1; - this.melstSystxEndYear = melstSystxEndYear != null ? melstSystxEndYear : -1; - this.melstSystxDaysDuration = melstSystxDaysDuration != null ? melstSystxDaysDuration : -1; - } - - public String getMELST_PTID() { - return melstPtid; - } - - public void setMELST_PTID(String melstPtid) { - this.melstPtid = melstPtid; - } - - public String getMELST_SYSTX_TYP_DESC() { - return melstSystxTypDesc; - } - - public void setMELST_SYSTX_TYP_DESC(String melstSystxTypeDesc) { - this.melstSystxTypDesc = melstSystxTypeDesc; - } - - public String getMELST_SYSTX_TYPE_OTH() { - return melstSystxTypeOth; - } - - public void setMELST_SYSTX_TYPE_OTH(String melstSystxTypeOth) { - this.melstSystxTypeOth = melstSystxTypeOth; - } - - public Integer getMELST_SYSTX_STRT_YEAR() { - return melstSystxStrtYear; - } - - public void setMELST_SYSTX_STRT_YEAR(Integer melstSystxStrtYear) { - this.melstSystxStrtYear = melstSystxStrtYear; - } - - public Integer getMELST_SYSTX_END_YEAR() { - return melstSystxEndYear; - } - - public void setMELST_SYSTX_END_YEAR(Integer melstSystxEndYear) { - this.melstSystxEndYear = melstSystxEndYear; - } - - public Integer getMELST_SYSTX_DAYS_DURATION() { - return melstSystxDaysDuration; - } - - public void setMELST_SYSTX_DAYS_DURATION(Integer melstSystxDaysDuration) { - this.melstSystxDaysDuration = melstSystxDaysDuration; - } - - public List getFieldNames() { - List fieldNames = new ArrayList<>(); - - fieldNames.add("PATIENT_ID"); - fieldNames.add("START_DATE"); - fieldNames.add("STOP_DATE"); - fieldNames.add("EVENT_TYPE"); - fieldNames.add("TREATMENT_TYPE"); - fieldNames.add("SUBTYPE"); - - return fieldNames; - } - - @Override - public String getPATIENT_ID() { - return melstPtid; - } - - // due to insufficient data in darwin, start date is 0 - @Override - public String getSTART_DATE() { - return "0"; - } - - @Override - public String getSTOP_DATE() { - return String.valueOf(melstSystxDaysDuration); - } - - @Override - public String getEVENT_TYPE() { - return "TREATMENT"; - } - - @Override - public String getTREATMENT_TYPE() { - return EXCLUDED_TREATMENTS.contains(melstSystxTypDesc) ? melstSystxTypeOth : melstSystxTypDesc; - } - - @Override - public String getSUBTYPE() { - return ""; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyClinicalWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyClinicalWriter.java deleted file mode 100644 index b7359b78e..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyClinicalWriter.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.mskimpact_medicaltherapy; - -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactMedicalTherapy; - -import com.google.common.base.Strings; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.core.io.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import java.io.*; -import java.util.*; - -/** - * @author Benjamin Gross - */ -public class MskimpactMedicalTherapyClinicalWriter implements ItemStreamWriter -{ - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.medicaltherapy_clinical_filename}") - private String outputFilename; - - private File stagingFile; - - private int recordsWritten; - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write("PATIENT_ID\tIMMUNE_TREATMENT"); - } - }); - stagingFile = new File(outputDirectory, outputFilename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{ - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception{ - writeList.clear(); - for (MskimpactMedicalTherapy item : items) { - if (!Strings.isNullOrEmpty(item.getDMP_ID_PHARMACY())) { - writeList.add(item.getDMP_ID_PHARMACY() + "\tYes"); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyProcessor.java deleted file mode 100644 index 9295b7bae..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyProcessor.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpact_medicaltherapy; - -import java.util.*; -import org.apache.commons.collections.map.MultiKeyMap; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactMedicalTherapy; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * This processor takes a list of medical therapy records all for a single patient, grouped by - * DMP-ID/drug-name/dosage/dose-unit/dispensed quantity and collapses them down into a single record - * with a start/stop date based on a sort of the AGE_AT_DISPENSE_DATE_IN_DAYS values across all records. - * - * @author Benjamin Gross - */ -public class MskimpactMedicalTherapyProcessor implements ItemProcessor, MskimpactMedicalTherapy> { - - @Autowired - private DarwinUtils darwinUtils; - - /** - * All items in the list are identical with the exception of AGE_AT_DISPENSE_DATE_IN_DAYS, the time at which - * the drug was dispensed to the patient. - */ - @Override - public MskimpactMedicalTherapy process(final List listMedicalTherapyRecords) throws Exception { - checkConsistencyOfRecords(listMedicalTherapyRecords); - Collections.sort(listMedicalTherapyRecords); - MskimpactMedicalTherapy oldestRecord = listMedicalTherapyRecords.get(0); - MskimpactMedicalTherapy youngestRecord = listMedicalTherapyRecords.get(listMedicalTherapyRecords.size()-1); - // we can use any record in the list to populate all fields except for START_DATE, STOP_DATE - return new MskimpactMedicalTherapy(darwinUtils.convertWhitespace(oldestRecord.getPT_ID_PHARMACY()), - darwinUtils.convertWhitespace(oldestRecord.getDMP_ID_PHARMACY()), - -1, - darwinUtils.convertWhitespace(oldestRecord.getGENERIC_DRUG_NAME()), - oldestRecord.getDOSAGE(), - darwinUtils.convertWhitespace(oldestRecord.getDOSE_UNIT()), - oldestRecord.getDISPENSED_QUANTITY(), - oldestRecord.getAGE_AT_DISPENSE_DATE_IN_DAYS(), - youngestRecord.getAGE_AT_DISPENSE_DATE_IN_DAYS(), - oldestRecord.getSAMPLE_ID_PATH_DMP()); - } - - private void checkConsistencyOfRecords(List listMedicalTherapyRecords) { - MultiKeyMap byRecordAttributes = new MultiKeyMap(); - for (MskimpactMedicalTherapy record : listMedicalTherapyRecords) { - byRecordAttributes.put(record.getDMP_ID_PHARMACY(), record.getGENERIC_DRUG_NAME(), - record.getDOSAGE(), record.getDOSE_UNIT(), record.getDISPENSED_QUANTITY()); - } - assert byRecordAttributes.keySet().size() == 1; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyReader.java deleted file mode 100644 index d3fb40b1e..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyReader.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.mskimpact_medicaltherapy; - -import org.mskcc.cmo.ks.darwin.pipeline.model.*; - -import com.querydsl.sql.SQLQueryFactory; -import com.querydsl.core.types.Projections; -import com.querydsl.core.BooleanBuilder; -import static com.querydsl.core.alias.Alias.*; - -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; - -import org.apache.log4j.Logger; -import org.apache.commons.collections.MapIterator; -import org.apache.commons.collections.map.MultiKeyMap; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.annotation.Resource; - -/** - * Code reads information out of DVCBIO.PHARMACY_V looking - * for records in which immunotherapy related drugs were dispensed. - * - * @author Benjamin Gross - */ -public class MskimpactMedicalTherapyReader implements ItemStreamReader> -{ - private String pharmacyView = "PHARMACY_V"; - - @Value("${darwin.pathology_dmp_view}") - private String pathologyView; - - @Value("${darwin.medicaltherapy.pharmacy_druglist}") - private String pharmacyViewDrugList; - - @Value("${darwin.medicaltherapy.skip_negative_dosage_records}") - private Boolean skipNegativeDosageRecords; - - @Value("#{jobParameters[studyID]}") - private String studyID; - - @Resource(name="studyIdRegexMap") - Map studyIdRegexMap; - - @Autowired - SQLQueryFactory darwinQueryFactory; - - private MultiKeyMap byPharmacyRecordAttributes; - - Logger log = Logger.getLogger(MskimpactMedicalTherapyReader.class); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - List pharmacyViewResults = getPharmacyViewResults(); - if (pharmacyViewResults == null || pharmacyViewResults.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Pharmacy Views"); - } - this.byPharmacyRecordAttributes = groupPharmacyViewResults(pharmacyViewResults); - } - - @Transactional - private List getPharmacyViewResults() { - - if (log.isInfoEnabled()) { - log.info("Start of " + pharmacyView + " record fetching..."); - } - - MskimpactMedicalTherapy pharmacyViewRecord = alias(MskimpactMedicalTherapy.class, pharmacyView); - MskimpactPathologyDmp pathologyViewRecord = alias(MskimpactPathologyDmp.class, pathologyView); - List pharmacyViewResults = - darwinQueryFactory.select(Projections.constructor(MskimpactMedicalTherapy.class, - $(pharmacyViewRecord.getPT_ID_PHARMACY()), - $(pharmacyViewRecord.getDMP_ID_PHARMACY()), - $(pharmacyViewRecord.getAGE_AT_DISPENSE_DATE_IN_DAYS()), - $(pharmacyViewRecord.getGENERIC_DRUG_NAME()), - $(pharmacyViewRecord.getDOSAGE()), - $(pharmacyViewRecord.getDOSE_UNIT()), - $(pharmacyViewRecord.getDISPENSED_QUANTITY()))) - .from($(pharmacyViewRecord)) - .innerJoin($(pathologyViewRecord)) - .on($(pharmacyViewRecord.getPT_ID_PHARMACY()).eq($(pathologyViewRecord.getPT_ID_PATH_DMP()))) - .where(getDrugClause(pharmacyViewRecord)) - .fetch(); - - List filteredPharmacyViewResults = new ArrayList<>(); - for (MskimpactMedicalTherapy result : pharmacyViewResults) { - Matcher matcher = studyIdRegexMap.get(studyID).matcher(result.getSAMPLE_ID_PATH_DMP()); - if (matcher.matches()) { - filteredPharmacyViewResults.add(result); - } - } - - if (log.isInfoEnabled()) { - log.info("Fetched " + filteredPharmacyViewResults.size() + " records from " + pharmacyView + "."); - } - - return filteredPharmacyViewResults; - } - - /** - * This method constructs part of the pharmacy view 'where' clause - * such that only records that contain a drug or drugs of interest (provided by application.properties) - * are returned from the query. - */ - private BooleanBuilder getDrugClause(MskimpactMedicalTherapy pharmacyViewRecord) { - BooleanBuilder builder = new BooleanBuilder(); - for (String drug : pharmacyViewDrugList.split(";")) { - builder.or($(pharmacyViewRecord.getGENERIC_DRUG_NAME()).toUpperCase().like("%" + drug + "%")); - } - return builder; - } - - /** - * This method collects all pharmacy records for a patient into a list group by - * DMP-ID/drug-name/dosage/dose-unit/dispensed quantity. - */ - private MultiKeyMap groupPharmacyViewResults(List pharmacyViewResults) { - MultiKeyMap toReturn = new MultiKeyMap(); - for (MskimpactMedicalTherapy pharmacyViewResult : pharmacyViewResults) { - if (skipNegativeDosageRecords && pharmacyViewResult.getDOSAGE() < 0) continue; - if (!toReturn.containsKey(pharmacyViewResult.getDMP_ID_PHARMACY(), - pharmacyViewResult.getGENERIC_DRUG_NAME(), - pharmacyViewResult.getDOSAGE(), - pharmacyViewResult.getDOSE_UNIT(), - pharmacyViewResult.getDISPENSED_QUANTITY())) { - List newMedicalTherapyList = new ArrayList(); - newMedicalTherapyList.add(pharmacyViewResult); - toReturn.put(pharmacyViewResult.getDMP_ID_PHARMACY(), - pharmacyViewResult.getGENERIC_DRUG_NAME(), - pharmacyViewResult.getDOSAGE(), - pharmacyViewResult.getDOSE_UNIT(), - pharmacyViewResult.getDISPENSED_QUANTITY(), - newMedicalTherapyList); - } - else { - List existingMedicalTherapyList = - (List)toReturn.get(pharmacyViewResult.getDMP_ID_PHARMACY(), - pharmacyViewResult.getGENERIC_DRUG_NAME(), - pharmacyViewResult.getDOSAGE(), - pharmacyViewResult.getDOSE_UNIT(), - pharmacyViewResult.getDISPENSED_QUANTITY()); - existingMedicalTherapyList.add(pharmacyViewResult); - toReturn.put(pharmacyViewResult.getDMP_ID_PHARMACY(), - pharmacyViewResult.getGENERIC_DRUG_NAME(), - pharmacyViewResult.getDOSAGE(), - pharmacyViewResult.getDOSE_UNIT(), - pharmacyViewResult.getDISPENSED_QUANTITY(), - existingMedicalTherapyList); - } - } - return toReturn; - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{} - - @Override - public List read() throws Exception { - MapIterator it = byPharmacyRecordAttributes.mapIterator(); - if (it.hasNext()) { - it.next(); // calling .next() allows us to make the subsequent getValue() call - List toReturn = (List)it.getValue(); - it.remove(); - return toReturn; - } - return null; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyTimelineWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyTimelineWriter.java deleted file mode 100644 index 1620a1600..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpact_medicaltherapy/MskimpactMedicalTherapyTimelineWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpact_medicaltherapy; - -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactMedicalTherapy; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.*; - -/** - * @author Benjamin Gross - */ -public class MskimpactMedicalTherapyTimelineWriter implements ItemStreamWriter -{ - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.medicaltherapy_timeline_filename}") - private String outputFilename; - - private String stagingFile; - - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write(String.join("\t", MskimpactMedicalTherapy.getHeaders())); - } - }); - stagingFile = outputDirectory + File.separator + outputFilename; - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{ - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception { - // TBD: figure out why we cannot pass items to FlatFileItemWriter directly - writeList.clear(); - for (MskimpactMedicalTherapy item : items) { - writeList.add(item.toString()); - } - flatFileItemWriter.write(writeList); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalProcessor.java deleted file mode 100644 index 03d2901c8..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalProcessor.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspineclinical; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineClinical; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author jake - */ -public class MskimpactBrainSpineClinicalProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - @Override - public String process(final MskimpactBrainSpineClinical darwinClinicalBrainSpine) throws Exception { - List record = new ArrayList<>(); - for(String field : new MskimpactBrainSpineClinical().getFieldNames()){ - String value = darwinClinicalBrainSpine.getClass().getMethod("get"+field).invoke(darwinClinicalBrainSpine).toString(); - record.add(darwinUtils.convertWhitespace(value)); - } - return String.join("\t", record); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalReader.java deleted file mode 100644 index 6b501eb6e..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalReader.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspineclinical; - -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineClinical; - -import com.querydsl.core.types.Projections; -import com.querydsl.sql.SQLQueryFactory; -import static com.querydsl.core.alias.Alias.*; - -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; -import org.apache.log4j.Logger; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.annotation.Resource; - -/** - * - * @author jake - */ -public class MskimpactBrainSpineClinicalReader implements ItemStreamReader{ - @Value("${darwin.clinical_view}") - private String clinicalBrainSpineView; - - @Value("#{jobParameters[studyID]}") - private String studyID; - - @Autowired - SQLQueryFactory darwinQueryFactory; - - @Resource(name="studyIdRegexMap") - Map studyIdRegexMap; - - private List clinicalBrainSpineResults; - - Logger log = Logger.getLogger(MskimpactBrainSpineClinicalReader.class); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - this.clinicalBrainSpineResults = getClinicalBrainSpineResults(); - if (clinicalBrainSpineResults == null || clinicalBrainSpineResults.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Brain Spine Clinical Views"); - } - } - - @Transactional - private List getClinicalBrainSpineResults(){ - log.info("Start of Clinical Brain Spine View Import..."); - MskimpactBrainSpineClinical qCBSR = alias(MskimpactBrainSpineClinical.class, clinicalBrainSpineView); - List clinicalBrainSpineResults = darwinQueryFactory.selectDistinct(Projections.constructor(MskimpactBrainSpineClinical.class, - $(qCBSR.getDMP_PATIENT_ID_BRAINSPINECLIN()), - $(qCBSR.getDMP_SAMPLE_ID_BRAINSPINECLIN()), - $(qCBSR.getAGE()), - $(qCBSR.getSEX()), - $(qCBSR.getOS_STATUS()), - $(qCBSR.getOS_MONTHS()), - $(qCBSR.getDFS_STATUS()), - $(qCBSR.getDFS_MONTHS()), - $(qCBSR.getHISTOLOGY()), - $(qCBSR.getWHO_GRADE()), - $(qCBSR.getMGMT_STATUS()))) - .where($(qCBSR.getDMP_PATIENT_ID_BRAINSPINECLIN()).isNotEmpty()) - .from($(qCBSR)) - .fetch(); - - List filteredClinicalBrainSpineResults = new ArrayList<>(); - for (MskimpactBrainSpineClinical result : clinicalBrainSpineResults) { - Matcher matcher = studyIdRegexMap.get(studyID).matcher(result.getDMP_SAMPLE_ID_BRAINSPINECLIN()); - if (matcher.matches()) { - filteredClinicalBrainSpineResults.add(result); - } - } - - log.info("Imported " + filteredClinicalBrainSpineResults.size() + " records from Clinical Brain Spine View."); - return filteredClinicalBrainSpineResults; - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{} - - @Override - public MskimpactBrainSpineClinical read() throws Exception{ - if(!clinicalBrainSpineResults.isEmpty()){ - return clinicalBrainSpineResults.remove(0); - } - return null; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalWriter.java deleted file mode 100644 index 8e59171b5..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspineclinical/MskimpactBrainSpineClinicalWriter.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspineclinical; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineClinical; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.*; - -/** - * - * @author jake - */ -public class MskimpactBrainSpineClinicalWriter implements ItemStreamWriter{ - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.brainspine_clinical_filename}") - private String datasetFilename; - - private int recordsWritten; - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write(String.join("\t", new MskimpactBrainSpineClinical().getHeaders())); - } - }); - stagingFile = new File(outputDirectory, datasetFilename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{ - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception{ - writeList.clear(); - for(String result : items){ - if (!Strings.isNullOrEmpty(result)) { - writeList.add(result); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/BrainSpineTimelineType.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/BrainSpineTimelineType.java deleted file mode 100644 index 5e8928cdc..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/BrainSpineTimelineType.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2016 - 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline; - -public enum BrainSpineTimelineType { - STATUS("Status"), - TREATMENT("Treatment"), - SURGERY("Surgery"), - SPECIMEN("Specimen"), - IMAGING("Imaging"); - - private String type; - - BrainSpineTimelineType(String type) { - this.type = type; - } - - public String toString() { - return type; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineCompositeToCompositeProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineCompositeToCompositeProcessor.java deleted file mode 100644 index 9851e0714..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineCompositeToCompositeProcessor.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineCompositeTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author jake - */ -public class MskimpactTimelineBrainSpineCompositeToCompositeProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - private BrainSpineTimelineType type; - - public MskimpactTimelineBrainSpineCompositeToCompositeProcessor(BrainSpineTimelineType type) { - this.type = type; - } - - @Override - public MskimpactBrainSpineCompositeTimeline process(final MskimpactBrainSpineCompositeTimeline composite) throws Exception{ - List recordPost = new ArrayList<>(); - for (String field : (List)composite.getRecord().getClass().getMethod("get" + type.toString() + "Fields").invoke(composite.getRecord())) { - String value = composite.getRecord().getClass().getMethod("get" + field).invoke(composite.getRecord()).toString(); - recordPost.add(darwinUtils.convertWhitespace(value)); - } - if(recordPost.contains(type.toString().toUpperCase())){ - composite.getClass().getMethod("set" + type.toString() + "Result", String.class).invoke(composite, String.join("\t", recordPost)); - } - return composite; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineModelToCompositeProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineModelToCompositeProcessor.java deleted file mode 100644 index 4a1310738..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineModelToCompositeProcessor.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2016, 2017 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineCompositeTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline.BrainSpineTimelineType; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author jake - */ -public class MskimpactTimelineBrainSpineModelToCompositeProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - private BrainSpineTimelineType type; - - public MskimpactTimelineBrainSpineModelToCompositeProcessor(BrainSpineTimelineType type) { - this.type = type; - } - - @Override - public MskimpactBrainSpineCompositeTimeline process(final MskimpactBrainSpineTimeline timelineBrainSpine) throws Exception { - MskimpactBrainSpineCompositeTimeline composite = new MskimpactBrainSpineCompositeTimeline(timelineBrainSpine); - List recordPost = new ArrayList<>(); - for (String field : (List)composite.getRecord().getClass().getMethod("get" + type.toString() + "Fields").invoke(composite.getRecord())) { - String value = composite.getRecord().getClass().getMethod("get" + field).invoke(composite.getRecord()).toString(); - recordPost.add(darwinUtils.convertWhitespace(value)); - } - if (recordPost.contains(type.toString().toUpperCase())) { - composite.getClass().getMethod("set" + type.toString() + "Result", String.class).invoke(composite, String.join("\t", recordPost)); - } - return composite; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineReader.java deleted file mode 100644 index 30a952730..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineReader.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2016, 2018, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline; - -import com.querydsl.core.types.Projections; -import com.querydsl.sql.SQLQueryFactory; -import java.util.*; -import org.apache.log4j.Logger; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinSampleListUtil; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.transaction.annotation.Transactional; -import static com.querydsl.core.alias.Alias.*; - -/** - * - * @author jake - */ -public class MskimpactTimelineBrainSpineReader implements ItemStreamReader{ - @Value("${darwin.timeline_view}") - private String timelineBrainSpineView; - - @Autowired - SQLQueryFactory darwinQueryFactory; - - @Autowired - public DarwinSampleListUtil darwinSampleListUtil; - - private List darwinTimelineResults; - - Logger log = Logger.getLogger(MskimpactTimelineBrainSpineReader.class); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - this.darwinTimelineResults = getDarwinTimelineResults(); - if (darwinTimelineResults == null || darwinTimelineResults.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Brain Spine Timeline Views"); - } - } - - // This method was added for unit testing of the read() method. An upgrade of mockito and jvp seems to not allow use of mocked objects (instantiated classes) into @Transactional blocks. - public void openForTestingAndSetDarwinTimelineResults(List testingDarwinTimelineResults) { - darwinTimelineResults = testingDarwinTimelineResults; - } - - @Transactional - private List getDarwinTimelineResults(){ - log.info("Start of Darwin Timeline Brain Spine View import..."); - MskimpactBrainSpineTimeline qDTR = alias(MskimpactBrainSpineTimeline.class, timelineBrainSpineView); - List darwinTimelineResults = darwinQueryFactory.selectDistinct(Projections.constructor(MskimpactBrainSpineTimeline.class, $(qDTR.getDMT_PATIENT_ID_BRAINSPINETMLN()), - $(qDTR.getDMP_PATIENT_ID_MIN_BRAINSPINETMLN()), - $(qDTR.getDMP_PATIENT_ID_MAX_BRAINSPINETMLN()), $(qDTR.getDMP_PATIENT_ID_COUNT_BRAINSPINETMLN()), - $(qDTR.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN()), $(qDTR.getSTART_DATE()), $(qDTR.getSTOP_DATE()), - $(qDTR.getEVENT_TYPE()), $(qDTR.getTREATMENT_TYPE()), $(qDTR.getSUBTYPE()), $(qDTR.getAGENT()), - $(qDTR.getSPECIMEN_REFERENCE_NUMBER()), $(qDTR.getSPECIMEN_SITE()), $(qDTR.getSPECIMEN_TYPE()), - $(qDTR.getSTATUS()), $(qDTR.getKARNOFSKY_PERFORMANCE_SCORE()), $(qDTR.getSURGERY_DETAILS()), - $(qDTR.getEVENT_TYPE_DETAILED()), $(qDTR.getHISTOLOGY()), $(qDTR.getWHO_GRADE()), - $(qDTR.getMGMT_STATUS()), $(qDTR.getSOURCE_PATHOLOGY()), $(qDTR.getNOTE()), $(qDTR.getDIAGNOSTIC_TYPE()), - $(qDTR.getDIAGNOSTIC_TYPE_DETAILED()), $(qDTR.getSOURCE()))) - .from($(qDTR)) - .where($(qDTR.getDMT_PATIENT_ID_BRAINSPINETMLN()).isNotNull()) - .fetch(); - - log.info("Imported " + darwinTimelineResults.size() + " records from Darwin Timeline Brain Spine View."); - return darwinTimelineResults; - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{} - - @Override - public MskimpactBrainSpineTimeline read() throws Exception{ - while(!darwinTimelineResults.isEmpty()){ - MskimpactBrainSpineTimeline darwinTimelineResult = darwinTimelineResults.remove(0); - if (darwinTimelineResult.getSTART_DATE().equals("NA")) { - darwinSampleListUtil.addFilteredMskimpactBrainSpineTimeline(darwinTimelineResult); - } else { - return darwinTimelineResult; - } - } - return null; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineWriter.java deleted file mode 100644 index f4b61a901..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactbrainspinetimeline/MskimpactTimelineBrainSpineWriter.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.apache.log4j.Logger; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineCompositeTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.*; - -/** - * - * @author jake - */ -public class MskimpactTimelineBrainSpineWriter implements ItemStreamWriter{ - private BrainSpineTimelineType type; - - public MskimpactTimelineBrainSpineWriter(BrainSpineTimelineType type) { - this.type = type; - } - - Logger log = Logger.getLogger(MskimpactTimelineBrainSpineWriter.class); - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - private int recordsWritten; - private List writeList = new ArrayList<>(); - private final FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - stagingFile = new File(outputDirectory, "data_timeline_" + type.toString().toLowerCase() + "_caisis_gbm.txt"); - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - MskimpactBrainSpineTimeline mskImpactBrainSpineTimeline = new MskimpactBrainSpineTimeline(); - try { - writer.write(String.join("\t", (List)mskImpactBrainSpineTimeline.getClass().getMethod("get" + type.toString() + "Headers").invoke(mskImpactBrainSpineTimeline))); - } - catch(Exception e) { - throw new ItemStreamException(e); - } - } - }); - - flatFileItemWriter.setResource(new FileSystemResource(this.stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{ - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception{ - writeList.clear(); - for (MskimpactBrainSpineCompositeTimeline result : items) { - String record = (String) result.getClass().getMethod("get" + type.toString() + "Result").invoke(result); - if (!Strings.isNullOrEmpty(record) && !record.equals(MskimpactBrainSpineCompositeTimeline.NO_RESULT)) { - writeList.add(record); - recordsWritten++; - } - } - if(!writeList.isEmpty()){ - flatFileItemWriter.write(writeList); - } - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactAgeWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactAgeWriter.java deleted file mode 100644 index 825df6974..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactAgeWriter.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.FileSystemResource; - -/** - * - * @author heinsz - */ -public class MskimpactAgeWriter implements ItemStreamWriter { - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.age_filename}") - private String datasetFilename; - - private int recordsWritten; - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write(String.join("\t", MskimpactPatientDemographics.getAgeHeaders())); - } - }); - stagingFile = new File(outputDirectory, datasetFilename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext ec) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException { - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception { - writeList.clear(); - for (MskimpactCompositeDemographics record : items) { - if (!Strings.isNullOrEmpty(record.getAgeResult())) { - writeList.add(record.getAgeResult()); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsProcessor.java deleted file mode 100644 index 0e9fc0f04..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsProcessor.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactCompositeDemographics; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactPatientDemographics; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.mskcc.cmo.ks.darwin.pipeline.util.VitalStatusUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author jake - */ -public class MskimpactPatientDemographicsProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - @Override - public MskimpactCompositeDemographics process(final MskimpactPatientDemographics darwinPatientDemographics) throws Exception{ - // format results for demographics, age, and survival status - List patientDemographicsResult = new ArrayList<>(); - for(String field : MskimpactPatientDemographics.getPatientDemographicsFieldNames()){ - Object value = darwinPatientDemographics.getClass().getMethod("get"+field).invoke(darwinPatientDemographics); - patientDemographicsResult.add(value != null ? darwinUtils.convertWhitespace(value.toString()) : "NA"); - } - List patientAgeResult = new ArrayList<>(); - for(String field : MskimpactPatientDemographics.getAgeFieldNames()){ - Object value = darwinPatientDemographics.getClass().getMethod("get"+field).invoke(darwinPatientDemographics); - patientAgeResult.add(value != null ? darwinUtils.convertWhitespace(value.toString()) : "NA"); - } - - List patientVitalStatusResult = VitalStatusUtils.getVitalStatusResult(darwinUtils, darwinPatientDemographics); - - MskimpactCompositeDemographics compositeRecord = new MskimpactCompositeDemographics(); - compositeRecord.setDemographicsResult(String.join("\t", patientDemographicsResult)); - compositeRecord.setAgeResult(String.join("\t", patientAgeResult)); - compositeRecord.setVitalStatusResult(String.join("\t", patientVitalStatusResult)); - return compositeRecord; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsReader.java deleted file mode 100644 index caf72b5c4..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsReader.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics; - -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import com.querydsl.core.types.Projections; -import com.querydsl.sql.SQLQueryFactory; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; -import org.apache.log4j.Logger; -import java.util.*; -import javax.annotation.Resource; -import static com.querydsl.core.alias.Alias.$; -import static com.querydsl.core.alias.Alias.alias; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -/** - * - * @author jake - */ -public class MskimpactPatientDemographicsReader implements ItemStreamReader{ - @Value("${darwin.demographics_view}") - private String patientDemographicsView; - - @Value("${darwin.icdo_view}") - private String patientIcdoView; - - @Value("${darwin.latest_activity_view}") - private String latestActivityView; - - @Value("${darwin.pathology_dmp_view}") - private String pathologyDmpView; - - @Value("${darwin.naaccr.ethnicity_mapping_table}") - private String naacrEthnicityMappingTable; - - @Value("${darwin.naaccr.race_mapping_table}") - private String naacrRaceMappingTable; - - @Value("${darwin.naaccr.sex_mapping_table}") - private String naacrSexMappingTable; - - @Value("#{jobParameters[studyID]}") - private String studyID; - - @Resource(name="studyIdRegexMap") - Map studyIdRegexMap; - - @Autowired - SQLQueryFactory darwinQueryFactory; - - private List darwinDemographicsResults; - private Set processedIds = new HashSet<>(); - private Integer missingTM_DX_YEAR = 0; - - Logger log = Logger.getLogger(MskimpactPatientDemographicsReader.class); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - this.darwinDemographicsResults = getDarwinDemographicsResults(); - if (darwinDemographicsResults == null || darwinDemographicsResults.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Demographics Views"); - } - } - - @Transactional - private List getDarwinDemographicsResults(){ - log.info("Start of Darwin Patient Demographics View Import..."); - - Map naaccrEthnicityMap = new HashMap<>(); - Map naaccrRaceMap = new HashMap<>(); - Map naaccrSexMap = new HashMap<>(); - - log.info("Query NAACR tables for code mappings..."); - NAACCREthnicityMapping qNAACCREthnicityMapping = alias(NAACCREthnicityMapping.class, naacrEthnicityMappingTable); - NAACCRRaceMapping qNAACCRRaceMapping = alias(NAACCRRaceMapping.class, naacrRaceMappingTable); - NAACCRSexMapping qNAACCRSexMapping = alias(NAACCRSexMapping.class, naacrSexMappingTable); - List naaccrEthnicityMappings = darwinQueryFactory.selectDistinct(Projections.constructor(NAACCREthnicityMapping.class, - $(qNAACCREthnicityMapping.getNECM_CODE()), - $(qNAACCREthnicityMapping.getNECM_CBIOPORTAL_LABEL()))) - .from($(qNAACCREthnicityMapping)) - .fetch(); - List naaccrRaceMappings = darwinQueryFactory.selectDistinct(Projections.constructor(NAACCRRaceMapping.class, - $(qNAACCRRaceMapping.getNRCM_CODE()), - $(qNAACCRRaceMapping.getNRCM_CBIOPORTAL_LABEL()))) - .from($(qNAACCRRaceMapping)) - .fetch(); - List naaccrSexMappings = darwinQueryFactory.selectDistinct(Projections.constructor(NAACCRSexMapping.class, - $(qNAACCRSexMapping.getNSCM_CODE()), - $(qNAACCRSexMapping.getNSCM_CBIOPORTAL_LABEL()))) - .from($(qNAACCRSexMapping)) - .fetch(); - - // Make maps out of the NAACCR lists for quick lookups - for (NAACCREthnicityMapping ethnicityMapping : naaccrEthnicityMappings) { - naaccrEthnicityMap.put(ethnicityMapping.getNECM_CODE(), ethnicityMapping.getNECM_CBIOPORTAL_LABEL()); - } - for (NAACCRRaceMapping raceMapping : naaccrRaceMappings) { - naaccrRaceMap.put(raceMapping.getNRCM_CODE(), raceMapping.getNRCM_CBIOPORTAL_LABEL()); - } - for (NAACCRSexMapping sexMapping : naaccrSexMappings) { - naaccrSexMap.put(sexMapping.getNSCM_CODE(), sexMapping.getNSCM_CBIOPORTAL_LABEL()); - } - - log.info("Query darwin tables for patient/sample demographics data..."); - MskimpactNAACCRClinical qMskimpactNAACCRClinical = alias(MskimpactNAACCRClinical.class, patientDemographicsView); - MskimpactPatientDemographics qMskImpactPatientDemographics = alias(MskimpactPatientDemographics.class, patientDemographicsView); - MskimpactPatientIcdoRecord qMskImpactPatientIcdoRecord = alias(MskimpactPatientIcdoRecord.class, patientIcdoView); - MskimpactLatestActivity qMskImpactLatestActivity = alias(MskimpactLatestActivity.class, latestActivityView); - MskimpactPathologyDmp qMskimpactPathologyDmp = alias(MskimpactPathologyDmp.class, pathologyDmpView); - List darwinDemographicsResultsList = darwinQueryFactory.selectDistinct(Projections.constructor(MskimpactPatientDemographics.class, - $(qMskImpactPatientDemographics.getDMP_ID_DEMO()), - $(qMskImpactPatientDemographics.getPT_NAACCR_SEX_CODE()), - $(qMskImpactPatientDemographics.getPT_NAACCR_RACE_CODE_PRIMARY()), - $(qMskImpactPatientDemographics.getPT_NAACCR_ETHNICITY_CODE()), - $(qMskImpactPatientDemographics.getRELIGION()), - $(qMskImpactPatientDemographics.getPT_VITAL_STATUS()), - $(qMskImpactPatientDemographics.getPT_BIRTH_YEAR()), - $(qMskImpactPatientDemographics.getPT_DEATH_YEAR()), - $(qMskImpactPatientIcdoRecord.getTM_DX_YEAR()), - $(qMskImpactLatestActivity.getAGE_AT_LAST_KNOWN_ALIVE_YEAR_IN_DAYS()), - $(qMskImpactPatientIcdoRecord.getAGE_AT_TM_DX_DATE_IN_DAYS()), - $(qMskImpactPatientDemographics.getAGE_AT_DATE_OF_DEATH_IN_DAYS()), - $(qMskImpactPatientDemographics.getPED_IND()), - $(qMskimpactPathologyDmp.getSAMPLE_ID_PATH_DMP()), - $(qMskImpactLatestActivity.getLAST_CONTACT_YEAR()), - $(qMskImpactLatestActivity.getAGE_AT_LAST_CONTACT_YEAR_IN_DAYS()))) - .from($(qMskImpactPatientDemographics)) - .fullJoin($(qMskImpactPatientIcdoRecord)) - .on($(qMskImpactPatientDemographics.getDMP_ID_DEMO()).eq($(qMskImpactPatientIcdoRecord.getDMP_ID_ICDO()))) - .fullJoin($(qMskImpactLatestActivity)) - .on($(qMskImpactPatientDemographics.getDMP_ID_DEMO()).eq($(qMskImpactLatestActivity.getDMP_ID_PLA()))) - .innerJoin($(qMskimpactPathologyDmp)) - .on($(qMskImpactPatientDemographics.getPT_ID_DEMO()).eq($(qMskimpactPathologyDmp.getPT_ID_PATH_DMP()))) - .orderBy($(qMskImpactPatientIcdoRecord.getTM_DX_YEAR()).asc()) - .fetch(); - - // Translate the NAACCR codes for each result - List filteredDarwinDemographicsResults = new ArrayList<>(); - for (MskimpactPatientDemographics result : darwinDemographicsResultsList) { - Matcher matcher = studyIdRegexMap.get(studyID).matcher(result.getSAMPLE_ID_PATH_DMP()); - if (!matcher.matches()) { - continue; - } - result.setGENDER(naaccrSexMap.getOrDefault(Integer.parseInt(result.getPT_NAACCR_SEX_CODE()), "NA")); - result.setRACE(naaccrRaceMap.getOrDefault(Integer.parseInt(result.getPT_NAACCR_RACE_CODE_PRIMARY()), "NA")); - result.setETHNICITY(naaccrEthnicityMap.getOrDefault(Integer.parseInt(result.getPT_NAACCR_ETHNICITY_CODE()), "NA")); - filteredDarwinDemographicsResults.add(result); - } - return filteredDarwinDemographicsResults; - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{} - - @Override - public MskimpactPatientDemographics read() throws Exception{ - return getRecord(); - } - - // Recursively remove items from darwinDemographicsResults looking for record to return. - // Patients can have multiple entries in the icdo view - we only want to return the one with the oldest year - // The records are ordered by TM_DX_YEAR - private MskimpactPatientDemographics getRecord() { - if (!darwinDemographicsResults.isEmpty()) { - MskimpactPatientDemographics record = darwinDemographicsResults.remove(0); - // Check if this patient has as already been processed. If it has, call this method again to find one that hasn't been. - if (!processedIds.contains(record.getDMP_ID_DEMO())) { - processedIds.add(record.getDMP_ID_DEMO()); - if(record.getTM_DX_YEAR().equals(-1)){ - missingTM_DX_YEAR++; - } - return record; - } - return getRecord(); - } - log.info("Imported " + processedIds.size() + " records from Demographics View."); - log.info(missingTM_DX_YEAR + " records missing TM_DX_YEAR!"); - return null; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsWriter.java deleted file mode 100644 index 8ed209d32..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactPatientDemographicsWriter.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactCompositeDemographics; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactPatientDemographics; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.*; - -/** - * - * @author jake - */ -public class MskimpactPatientDemographicsWriter implements ItemStreamWriter{ - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("#{jobParameters[currentDemographicsRecCount]}") - private Integer currentDemographicsRecCount; - - @Value("${darwin.demographics_filename}") - private String datasetFilename; - - private final double DEMOGRAPHIC_RECORD_DROP_THRESHOLD = 0.9; - private int recordsWritten; - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException{ - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write(String.join("\t", MskimpactPatientDemographics.getPatientDemographicsHeaders())); - } - }); - stagingFile = new File(outputDirectory, datasetFilename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{ - if (recordsWritten < (DEMOGRAPHIC_RECORD_DROP_THRESHOLD * currentDemographicsRecCount)) { - throw new RuntimeException("Number of records in latest demographics fetch (" + recordsWritten + - ") dropped greater than 90% of current record count (" + currentDemographicsRecCount + - ") in backup demographics file - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception{ - writeList.clear(); - for(MskimpactCompositeDemographics record : items){ - if (!Strings.isNullOrEmpty(record.getDemographicsResult())) { - writeList.add(record.getDemographicsResult()); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactVitalStatusWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactVitalStatusWriter.java deleted file mode 100644 index 61bc72272..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactdemographics/MskimpactVitalStatusWriter.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactCompositeDemographics; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactPatientDemographics; -import org.springframework.batch.item.*; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.FileSystemResource; - -/** - * - * @author ochoaa - */ -public class MskimpactVitalStatusWriter implements ItemStreamWriter { - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.vital_status_filename}") - private String datasetFilename; - - private int recordsWritten; - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write(String.join("\t", MskimpactPatientDemographics.getVitalStatusHeaders())); - } - }); - stagingFile = new File(outputDirectory, datasetFilename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException { - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception { - writeList.clear(); - for(MskimpactCompositeDemographics record : items){ - if (!Strings.isNullOrEmpty(record.getVitalStatusResult())) { - writeList.add(record.getVitalStatusResult()); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } - - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientProcessor.java deleted file mode 100644 index 2af26eb55..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientProcessor.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactgeniepatient; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactNAACCRClinical; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author heinsz - */ -public class MskimpactGeniePatientProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - @Override - public String process(MskimpactNAACCRClinical mskimpactNAACCRClinical) throws Exception { - List record = new ArrayList<>(); - for (String field : mskimpactNAACCRClinical.getFieldNames()) { - String value = mskimpactNAACCRClinical.getClass().getMethod("get" + field).invoke(mskimpactNAACCRClinical).toString(); - record.add(darwinUtils.convertWhitespace(value)); - } - return String.join("\t", record); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientReader.java deleted file mode 100644 index 55a2d4aa3..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientReader.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2016-2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactgeniepatient; - -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactNAACCRClinical; -import org.springframework.batch.item.*; -import com.querydsl.core.types.Projections; -import com.querydsl.sql.SQLQueryFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.annotation.Autowired; -import org.apache.log4j.Logger; -import java.util.*; -import static com.querydsl.core.alias.Alias.$; -import static com.querydsl.core.alias.Alias.alias; - -/** - * - * @author heinsz - */ -public class MskimpactGeniePatientReader implements ItemStreamReader { - @Value("${darwin.demographics_view}") - private String patientDemographicsView; - - @Autowired - SQLQueryFactory darwinQueryFactory; - - private List darwinDemographicsResults; - - Logger log = Logger.getLogger(MskimpactGeniePatientReader.class); - - @Override - public void open(ExecutionContext ec) throws ItemStreamException { - this.darwinDemographicsResults = getDarwinDemographicsResults(); - if (darwinDemographicsResults == null || darwinDemographicsResults.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Demographics Views"); - } - } - - @Override - public void update(ExecutionContext ec) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException {} - - @Override - public MskimpactNAACCRClinical read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { - if (!darwinDemographicsResults.isEmpty()) { - return darwinDemographicsResults.remove(0); - } - return null; - } - - private List getDarwinDemographicsResults() { - log.info("Start of Darwin NAACCR View Import..."); - MskimpactNAACCRClinical qMskimpactNAACCRClinical = alias(MskimpactNAACCRClinical.class, patientDemographicsView); - List darwinDemographicsResultsList = darwinQueryFactory.selectDistinct(Projections.constructor(MskimpactNAACCRClinical.class, - $(qMskimpactNAACCRClinical.getDMP_ID_DEMO()), - $(qMskimpactNAACCRClinical.getPT_NAACCR_SEX_CODE()), - $(qMskimpactNAACCRClinical.getPT_NAACCR_RACE_CODE_PRIMARY()), - $(qMskimpactNAACCRClinical.getPT_NAACCR_RACE_CODE_SECONDARY()), - $(qMskimpactNAACCRClinical.getPT_NAACCR_RACE_CODE_TERTIARY()), - $(qMskimpactNAACCRClinical.getPT_NAACCR_ETHNICITY_CODE()), - $(qMskimpactNAACCRClinical.getPT_BIRTH_YEAR()))) - .from($(qMskimpactNAACCRClinical)) - .fetch(); - return darwinDemographicsResultsList; - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientWriter.java deleted file mode 100644 index 4513b6ffe..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/mskimpactgeniepatient/MskimpactGeniePatientWriter.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.mskimpactgeniepatient; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactNAACCRClinical; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.FileSystemResource; - -/** - * - * @author heinsz - */ -public class MskimpactGeniePatientWriter implements ItemStreamWriter{ - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.genie_patient_filename}") - private String datasetFilename; - - private int recordsWritten; - private List writeList = new ArrayList<>(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException{ - writer.write(String.join("\t", new MskimpactNAACCRClinical().getHeaders())); - } - }); - stagingFile = new File(outputDirectory, datasetFilename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext ec) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException { - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List list) throws Exception { - writeList.clear(); - for (String record : list) { - if (!Strings.isNullOrEmpty(record)) { - writeList.add(record); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientProcessor.java deleted file mode 100644 index 092ea9d24..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientProcessor.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chantclinical; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalPatientProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - @Value("#{stepExecutionContext['patientHeader']}") - private Map> patientHeader; - - @Override - public Skcm_mskcc_2015_chantClinicalCompositeRecord process(final Skcm_mskcc_2015_chantClinicalCompositeRecord melanomaClinicalCompositeRecord) throws Exception { - Skcm_mskcc_2015_chantNormalizedClinicalRecord melanomaClinicalRecord = melanomaClinicalCompositeRecord.getRecord(); - List header = melanomaClinicalRecord.getFieldNames(); - List record = new ArrayList<>(); - - // first add patient id to record then iterate through rest of patient header - record.add(darwinUtils.convertWhitespace(melanomaClinicalRecord.getPATIENT_ID()).split("\\|")[0]); - for (int i=0; i uniqueValues = new HashSet(Arrays.asList(darwinUtils.convertWhitespace(value).split("\\|"))); - List values = Arrays.asList(value.split("\\|")); - if (uniqueValues.size() == 1) { - record.add(values.get(0)); - } - else { - record.add(value); - } - } - melanomaClinicalCompositeRecord.setPatientRecord(String.join("\t", record)); - return melanomaClinicalCompositeRecord; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientWriter.java deleted file mode 100644 index 34b61e85b..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalPatientWriter.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chantclinical; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.*; -import org.springframework.core.io.*; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalPatientWriter implements ItemStreamWriter { - - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.skcm_mskcc_2015_chant_clinical_patient_filename}") - private String filename; - - @Value("#{stepExecutionContext['patientHeader']}") - private Map> patientHeader; - - private int recordsWritten; - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - this.stagingFile = new File(outputDirectory, filename); - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException { - writer.write("#" + getMetaLine(patientHeader.get("display_names")) + "\n"); - writer.write("#" + getMetaLine(patientHeader.get("descriptions")) + "\n"); - writer.write("#" + getMetaLine(patientHeader.get("datatypes")) + "\n"); - writer.write("#" + getMetaLine(patientHeader.get("priorities")) + "\n"); - writer.write(getMetaLine(patientHeader.get("header"))); - } - }); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException { - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); - for (Skcm_mskcc_2015_chantClinicalCompositeRecord result : items) { - if (!Strings.isNullOrEmpty(result.getPatientRecord())) { - writeList.add(result.getPatientRecord()); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } - - private String getMetaLine(List metaData) { - int pidIndex = patientHeader.get("header").indexOf("PATIENT_ID"); - return metaData.remove(pidIndex) + "\t" + String.join("\t", metaData); - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalReader.java deleted file mode 100644 index 57331c507..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalReader.java +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chantclinical; - -import com.querydsl.core.types.Projections; -import com.querydsl.sql.SQLQueryFactory; -import java.lang.reflect.Method; -import java.util.*; -import org.apache.log4j.Logger; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantClinicalRecord; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantNormalizedClinicalRecord; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics.MskimpactPatientDemographicsReader; -import org.mskcc.cmo.ks.redcap.source.ClinicalDataSource; -import org.mskcc.cmo.ks.redcap.source.MetadataManager; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.*; -import org.springframework.transaction.annotation.Transactional; -import static com.querydsl.core.alias.Alias.*; -import static com.querydsl.core.alias.Alias.alias; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalReader implements ItemStreamReader{ - - @Value("${darwin.skcm_mskcc_2015_chant.staging_path_current_view}") - private String stagingPathCurrentView; - - @Value("${darwin.skcm_mskcc_2015_chant.primary_view}") - private String primaryView; - - @Value("${darwin.skcm_mskcc_2015_chant.general_view}") - private String generalView; - - @Value("${darwin.skcm_mskcc_2015_chant.impact_view}") - private String impactView; - - @Value("${darwin.skcm_mskcc_2015_chant.metastat_sites_view}") - private String metastatSitesView; - - @Autowired - public MetadataManager metadataManager; - - @Autowired - public ClinicalDataSource clinicalDataSource; - - @Autowired - SQLQueryFactory darwinQueryFactory; - - Logger log = Logger.getLogger(MskimpactPatientDemographicsReader.class); - - private List melanomaClinicalRecords; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - // add the patient and sample headers to the execution context - Map> fullHeader = metadataManager.getFullHeader(new Skcm_mskcc_2015_chantNormalizedClinicalRecord().getFieldNames()); - executionContext.put("sampleHeader", metadataManager.getFullSampleHeader(fullHeader)); - executionContext.put("patientHeader", metadataManager.getFullPatientHeader(fullHeader)); - - // getting records from db view and merge data by sample id - this.melanomaClinicalRecords = getMelanomaClinicalRecords(); - if (melanomaClinicalRecords == null || melanomaClinicalRecords.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Melanoma Clinical Views"); - } - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException{} - - @Override - public void close() throws ItemStreamException{} - - @Override - public Skcm_mskcc_2015_chantNormalizedClinicalRecord read() throws Exception{ - if(!melanomaClinicalRecords.isEmpty()) { - return melanomaClinicalRecords.remove(0); - } - return null; - } - - private List getMelanomaClinicalRecords() { - log.info("Start of Darwin skcm_mskcc_2015_chant clinical records query.."); - List rawDarwinMelanomaRecords = getRecordsFromDatabase(); - List combinedRecords = combineRecords(rawDarwinMelanomaRecords); - List normalizedRecords = normalizeRecords(combinedRecords); - return normalizedRecords; - } - - @Transactional - private List getRecordsFromDatabase() { - Skcm_mskcc_2015_chantClinicalRecord qStagingPathCurrentView = alias(Skcm_mskcc_2015_chantClinicalRecord.class, stagingPathCurrentView); - Skcm_mskcc_2015_chantClinicalRecord qPrimaryView = alias(Skcm_mskcc_2015_chantClinicalRecord.class, primaryView); - Skcm_mskcc_2015_chantClinicalRecord qGeneralView = alias(Skcm_mskcc_2015_chantClinicalRecord.class, generalView); - Skcm_mskcc_2015_chantClinicalRecord qImpactView = alias(Skcm_mskcc_2015_chantClinicalRecord.class, impactView); - Skcm_mskcc_2015_chantClinicalRecord qMetastatSitesView = alias(Skcm_mskcc_2015_chantClinicalRecord.class, metastatSitesView); - - List darwinMelanomaRecords = darwinQueryFactory.selectDistinct(Projections.constructor(Skcm_mskcc_2015_chantClinicalRecord.class, - $(qStagingPathCurrentView.getMELSPC_PTID()), - $(qStagingPathCurrentView.getMELSPC_STAGE_YEAR()), - $(qStagingPathCurrentView.getMELSPC_STG_GRP_NAME()), - $(qGeneralView.getMELG_PTID()), - $(qGeneralView.getMELG_STS_DESC()), - $(qGeneralView.getMELG_STS_SRCE_DESC()), - $(qGeneralView.getMELG_ACTV_STS_DESC()), - $(qGeneralView.getMELG_DERMAGRPHX_DESC()), - $(qGeneralView.getMELG_PRES_STG_YEAR()), - $(qGeneralView.getMELG_FAMILY_HX_DESC()), - $(qGeneralView.getMELG_1ST_RECUR_YEAR()), - $(qGeneralView.getMELG_LOCAL_DESC()), - $(qGeneralView.getMELG_NODAL_DESC()), - $(qGeneralView.getMELG_INTRANSIT_DESC()), - $(qGeneralView.getMELG_SYS_DESC()), - $(qGeneralView.getMELG_RECUR_NDSZ_DESC()), - $(qGeneralView.getMELG_RECUR_NODAL_NO()), - $(qGeneralView.getMELG_LDH()), - $(qGeneralView.getMELG_LDH_YEAR()), - $(qGeneralView.getMELG_METS_DESC()), - $(qGeneralView.getMELG_ADJVNT_TX_DESC()), - $(qGeneralView.getMELG_SYS_TX_DESC()), - $(qGeneralView.getMELG_RAD_TX_DESC()), - $(qGeneralView.getMELG_SURG_DESC()), - $(qGeneralView.getMELG_TISSUE_BANK_AVAIL()), - $(qPrimaryView.getMELP_PTID()), - $(qPrimaryView.getMELP_PRIM_SEQ()), - $(qPrimaryView.getMELP_DX_YEAR()), - $(qPrimaryView.getMELP_MSK_REVIEW_DESC()), - $(qPrimaryView.getMELP_THICKNESS_MM()), - $(qPrimaryView.getMELP_CLARK_LVL_DESC()), - $(qPrimaryView.getMELP_ULCERATION_DESC()), - $(qPrimaryView.getMELP_SITE_DESC()), - $(qPrimaryView.getMELP_SUB_SITE_DESC()), - $(qPrimaryView.getMELP_TILS_DESC()), - $(qPrimaryView.getMELP_REGRESSION_DESC()), - $(qPrimaryView.getMELP_MARGINS_DESC()), - $(qPrimaryView.getMELP_MITIDX_UNK_DESC()), - $(qPrimaryView.getMELP_HIST_TYPE_DESC()), - $(qPrimaryView.getMELP_SATELLITES_DESC()), - $(qPrimaryView.getMELP_EXT_SLIDES_DESC()), - $(qPrimaryView.getMELP_LNORG_DX_DESC()), - $(qPrimaryView.getMELP_LNCLIN_STS_DESC()), - $(qPrimaryView.getMELP_LNSENTINBX_DESC()), - $(qPrimaryView.getMELP_LNSENTINBX_YEAR()), - $(qPrimaryView.getMELP_LNPROLYSCT_DESC()), - $(qPrimaryView.getMELP_LNPROSUCC_DESC()), - $(qPrimaryView.getMELP_LNDSCT_CMP_DESC()), - $(qPrimaryView.getMELP_LNDSCT_YEAR()), - $(qPrimaryView.getMELP_LNMATTED_DESC()), - $(qPrimaryView.getMELP_LNEXTNODST_DESC()), - $(qPrimaryView.getMELP_LNINTRMETS_DESC()), - $(qPrimaryView.getMELP_LNSIZE()), - $(qPrimaryView.getMELP_LNSIZE_UNK_DESC()), - $(qPrimaryView.getMELP_LNSLNLARG_SIZE()), - $(qPrimaryView.getMELP_LNIHC_DESC()), - $(qPrimaryView.getMELP_LNIMM_S100_DESC()), - $(qPrimaryView.getMELP_LNIMMHMB45_DESC()), - $(qPrimaryView.getMELP_LNIMM_MELA_DESC()), - $(qImpactView.getMELI_PTID()), - $(qImpactView.getMELI_DMP_PATIENT_ID()), - $(qImpactView.getMELI_DMP_SAMPLE_ID()), - $(qImpactView.getMELI_REPORT_YEAR()), - $(qImpactView.getMELI_PROCEDURE_YEAR()), - $(qImpactView.getMELI_TUMOR_TYPE()), - $(qImpactView.getMELI_PRIMARY_SITE()), - $(qImpactView.getMELI_MET_SITE()), - $(qMetastatSitesView.getMELMS_PTID()), - $(qMetastatSitesView.getMELMS_SITE_TYPE_DESC()), - $(qMetastatSitesView.getMELMS_SITE_DESC()), - $(qMetastatSitesView.getMELMS_SITE_YEAR()))) - .from($(qStagingPathCurrentView)) - .fullJoin($(qPrimaryView)) - .on($(qStagingPathCurrentView.getMELSPC_PTID()).eq($(qPrimaryView.getMELP_PTID()))) - .fullJoin($(qGeneralView)) - .on($(qStagingPathCurrentView.getMELSPC_PTID()).eq($(qGeneralView.getMELG_PTID())).or($(qPrimaryView.getMELP_PTID()).eq($(qGeneralView.getMELG_PTID())))) - .fullJoin($(qImpactView)) - .on($(qStagingPathCurrentView.getMELSPC_PTID()).eq($(qImpactView.getMELI_PTID())).or($(qPrimaryView.getMELP_PTID()).eq($(qImpactView.getMELI_PTID())).or($(qGeneralView.getMELG_PTID()).eq($(qImpactView.getMELI_PTID()))))) - .fullJoin($(qMetastatSitesView)) - .on($(qGeneralView.getMELG_PTID()).eq($(qMetastatSitesView.getMELMS_PTID())).or($(qPrimaryView.getMELP_PTID()).eq($(qMetastatSitesView.getMELMS_PTID()))).or($(qStagingPathCurrentView.getMELSPC_PTID()).eq($(qMetastatSitesView.getMELMS_PTID())))) - .fetch(); - return darwinMelanomaRecords; - } - - private List combineRecords(List records) { - Map recordMap = new HashMap(); - List combinedRecords = new ArrayList<>(); - - for (Skcm_mskcc_2015_chantClinicalRecord record : records) { - if (!recordMap.containsKey(record.getSAMPLE_ID())) { - recordMap.put(record.getSAMPLE_ID(), record); - } - else { - Skcm_mskcc_2015_chantClinicalRecord existingRecord = recordMap.get(record.getSAMPLE_ID()); - recordMap.put(record.getSAMPLE_ID(), combineRecords(existingRecord, record)); - } - } - combinedRecords.addAll(recordMap.values()); - return combinedRecords; - } - - private Skcm_mskcc_2015_chantClinicalRecord combineRecords(Skcm_mskcc_2015_chantClinicalRecord record1, Skcm_mskcc_2015_chantClinicalRecord record2) { - Skcm_mskcc_2015_chantClinicalRecord combined = new Skcm_mskcc_2015_chantClinicalRecord(); - try { - for (String field : combined.getAllVariables()) { - Method fieldGetter = combined.getClass().getMethod("get" + field); - Set values = new HashSet<>(); - values.add((String) fieldGetter.invoke(record1)); - values.add((String) fieldGetter.invoke(record2)); - combined.getClass().getMethod("set" + field, String.class).invoke(combined, String.join("|", values)); - } - } - catch (Exception e) { - log.error("Failed to combine records!" + e.getMessage()); - throw new ItemStreamException(e); - } - return combined; - } - - private List normalizeRecords(List records) { - List normalizedRecords = new ArrayList<>(); - for (Skcm_mskcc_2015_chantClinicalRecord record : records) { - Skcm_mskcc_2015_chantNormalizedClinicalRecord normalizedRecord = new Skcm_mskcc_2015_chantNormalizedClinicalRecord( - record.getPATIENT_ID(), - record.getSAMPLE_ID(), - record.getMELSPC_STAGE_YEAR(), - record.getMELSPC_STG_GRP_NAME(), - record.getMELG_STS_DESC(), - record.getMELG_STS_SRCE_DESC(), - record.getMELG_ACTV_STS_DESC(), - record.getMELG_DERMAGRPHX_DESC(), - record.getMELG_PRES_STG_YEAR(), - record.getMELG_FAMILY_HX_DESC(), - record.getMELG_1ST_RECUR_YEAR(), - record.getMELG_LOCAL_DESC(), - record.getMELG_NODAL_DESC(), - record.getMELG_INTRANSIT_DESC(), - record.getMELG_SYS_DESC(), - record.getMELG_RECUR_NDSZ_DESC(), - record.getMELG_RECUR_NODAL_NO(), - record.getMELG_LDH(), - record.getMELG_LDH_YEAR(), - record.getMELG_METS_DESC(), - record.getMELG_ADJVNT_TX_DESC(), - record.getMELG_SYS_TX_DESC(), - record.getMELG_RAD_TX_DESC(), - record.getMELG_SURG_DESC(), - record.getMELG_TISSUE_BANK_AVAIL(), - record.getMELP_PRIM_SEQ(), - record.getMELP_DX_YEAR(), - record.getMELP_MSK_REVIEW_DESC(), - record.getMELP_THICKNESS_MM(), - record.getMELP_CLARK_LVL_DESC(), - record.getMELP_ULCERATION_DESC(), - record.getMELP_SITE_DESC(), - record.getMELP_SUB_SITE_DESC(), - record.getMELP_TILS_DESC(), - record.getMELP_REGRESSION_DESC(), - record.getMELP_MARGINS_DESC(), - record.getMELP_MITIDX_UNK_DESC(), - record.getMELP_HIST_TYPE_DESC(), - record.getMELP_SATELLITES_DESC(), - record.getMELP_EXT_SLIDES_DESC(), - record.getMELP_LNORG_DX_DESC(), - record.getMELP_LNCLIN_STS_DESC(), - record.getMELP_LNSENTINBX_DESC(), - record.getMELP_LNSENTINBX_YEAR(), - record.getMELP_LNPROLYSCT_DESC(), - record.getMELP_LNPROSUCC_DESC(), - record.getMELP_LNDSCT_CMP_DESC(), - record.getMELP_LNDSCT_YEAR(), - record.getMELP_LNMATTED_DESC(), - record.getMELP_LNEXTNODST_DESC(), - record.getMELP_LNINTRMETS_DESC(), - record.getMELP_LNSIZE(), - record.getMELP_LNSIZE_UNK_DESC(), - record.getMELP_LNSLNLARG_SIZE(), - record.getMELP_LNIHC_DESC(), - record.getMELP_LNIMM_S100_DESC(), - record.getMELP_LNIMMHMB45_DESC(), - record.getMELP_LNIMM_MELA_DESC(), - record.getMELI_REPORT_YEAR(), - record.getMELI_PROCEDURE_YEAR(), - record.getMELI_TUMOR_TYPE(), - record.getMELI_PRIMARY_SITE(), - record.getMELI_MET_SITE(), - record.getMELMS_SITE_TYPE_DESC(), - record.getMELMS_SITE_DESC(), - record.getMELMS_SITE_YEAR()); - normalizedRecords.add(normalizedRecord); - } - return normalizedRecords; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleProcessor.java deleted file mode 100644 index 68802292e..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleProcessor.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2016, 2017, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chantclinical; - -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalSampleProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - @Value("#{stepExecutionContext['sampleHeader']}") - private Map> sampleHeader; - - @Override - public Skcm_mskcc_2015_chantClinicalCompositeRecord process(final Skcm_mskcc_2015_chantNormalizedClinicalRecord melanomaClinicalRecord) throws Exception { - List record = new ArrayList<>(); - // first add sample and patient id to record then iterate through rest of sample header - record.add(darwinUtils.convertWhitespace(melanomaClinicalRecord.getSAMPLE_ID().split("\\|")[0])); - record.add(darwinUtils.convertWhitespace(melanomaClinicalRecord.getPATIENT_ID().split("\\|")[0])); - for (int i=0; i uniqueValues = new HashSet(Arrays.asList(darwinUtils.convertWhitespace(value).split("\\|"))); - List values = Arrays.asList(value.split("\\|")); - if (uniqueValues.size() == 1) { - record.add(values.get(0)); - } - else { - record.add(value); - } - } - Skcm_mskcc_2015_chantClinicalCompositeRecord compositeRecord = new Skcm_mskcc_2015_chantClinicalCompositeRecord(melanomaClinicalRecord); - compositeRecord.setSampleRecord(String.join("\t", record)); - return compositeRecord; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleWriter.java deleted file mode 100644 index 629d8431b..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chantclinical/Skcm_mskcc_2015_chantClinicalSampleWriter.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chantclinical; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.*; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.*; -import org.springframework.core.io.*; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantClinicalSampleWriter implements ItemStreamWriter { - - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.skcm_mskcc_2015_chant_clinical_sample_filename}") - private String filename; - - @Value("#{stepExecutionContext['sampleHeader']}") - private Map> sampleHeader; - - @Value("#{stepExecutionContext['patientHeader']}") - private Map> patientHeader; - - private int recordsWritten; - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter<>(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - this.stagingFile = new File(outputDirectory, filename); - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException { - writer.write("#" + getMetaLine(sampleHeader.get("display_names"), patientHeader.get("display_names")) + "\n"); - writer.write("#" + getMetaLine(sampleHeader.get("descriptions"), patientHeader.get("descriptions")) + "\n"); - writer.write("#" + getMetaLine(sampleHeader.get("datatypes"), patientHeader.get("datatypes")) + "\n"); - writer.write("#" + getMetaLine(sampleHeader.get("priorities"), patientHeader.get("priorities")) + "\n"); - writer.write(getMetaLine(sampleHeader.get("header"), patientHeader.get("header"))); - } - }); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException { - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); - for (Skcm_mskcc_2015_chantClinicalCompositeRecord result : items) { - if (!Strings.isNullOrEmpty(result.getSampleRecord())) { - writeList.add(result.getSampleRecord()); - recordsWritten++; - } - } - - flatFileItemWriter.write(writeList); - } - private String getMetaLine(List sampleMetadata, List patientMetadata) { - int sidIndex = sampleHeader.get("header").indexOf("SAMPLE_ID"); - int pidIndex = patientHeader.get("header").indexOf("PATIENT_ID"); - return sampleMetadata.remove(sidIndex) + "\t" + patientMetadata.get(pidIndex) + "\t" + String.join("\t", sampleMetadata); - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineProcessor.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineProcessor.java deleted file mode 100644 index 93c44fc4d..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineProcessor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2016, 2017 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chanttimeline; - -import java.util.ArrayList; -import java.util.List; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantTimelineRecord; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinUtils; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantTimelineProcessor implements ItemProcessor { - - @Autowired - private DarwinUtils darwinUtils; - - @Override - public String process(final Skcm_mskcc_2015_chantTimelineRecord melanomaTimelineRecord) throws Exception { - List record = new ArrayList<>(); - for (String field : melanomaTimelineRecord.getFieldNames()) { - String value = melanomaTimelineRecord.getClass().getMethod("get" + field).invoke(melanomaTimelineRecord).toString(); - value = darwinUtils.convertWhitespace(value); - if (field.equals("STOP_DATE") && value.equals("-1")) { - value = ""; - } - record.add(value); - } - return String.join("\t", record); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineReader.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineReader.java deleted file mode 100644 index d2fa6b094..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineReader.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chanttimeline; - -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantTimelineRadTherapy; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantTimelineRecord; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantTimelineAdjuvantTx; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantTimelineSystemicTx; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactdemographics.MskimpactPatientDemographicsReader; -import java.util.*; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.batch.item.ItemStreamReader; -import com.querydsl.core.types.Projections; -import static com.querydsl.core.alias.Alias.*; -import com.querydsl.sql.SQLQueryFactory; -import org.apache.log4j.Logger; - - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantTimelineReader implements ItemStreamReader { - - @Value("${darwin.skcm_mskcc_2015_chant.adjuvant_tx_view}") - private String adjuvantTxView; - @Value("${darwin.skcm_mskcc_2015_chant.rad_therapy_view}") - private String radTherapyView; - @Value("${darwin.skcm_mskcc_2015_chant.systemic_tx_view}") - private String systemicTxView; - @Autowired - SQLQueryFactory darwinQueryFactory; - - List melanomaTimelineRecords; - Logger log = Logger.getLogger(MskimpactPatientDemographicsReader.class); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - this.melanomaTimelineRecords = getMelanomaTimelineRecords(); - if (melanomaTimelineRecords == null || melanomaTimelineRecords.isEmpty()) { - throw new ItemStreamException("Error fetching records from Darwin Melanoma Timeline Views"); - } - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException {} - - @Override - public Skcm_mskcc_2015_chantTimelineRecord read() throws Exception { - if (!melanomaTimelineRecords.isEmpty()) { - return melanomaTimelineRecords.remove(0); - } - return null; - } - - @Transactional - private List getMelanomaTimelineRecords() { - log.info("Start of Darwin skcm_mskcc_2015_chant timeline records query.."); - List darwinRecords = new ArrayList<>(); - Skcm_mskcc_2015_chantTimelineAdjuvantTx qAdjuvantTxView = alias(Skcm_mskcc_2015_chantTimelineAdjuvantTx.class, adjuvantTxView); - Skcm_mskcc_2015_chantTimelineRadTherapy qRadTherapyView = alias(Skcm_mskcc_2015_chantTimelineRadTherapy.class, radTherapyView); - Skcm_mskcc_2015_chantTimelineSystemicTx qSystemicTxView = alias(Skcm_mskcc_2015_chantTimelineSystemicTx.class, systemicTxView); - - List adjuvantTimelineRecords = darwinQueryFactory.selectDistinct(Projections.constructor(Skcm_mskcc_2015_chantTimelineAdjuvantTx.class, - $(qAdjuvantTxView.getMELAT_PTID()), - $(qAdjuvantTxView.getMELAT_ADJTX_TYP_CD()), - $(qAdjuvantTxView.getMELAT_ADJTX_TYP_DESC()), - $(qAdjuvantTxView.getMELAT_ADJTX_TYPE_OTH()), - $(qAdjuvantTxView.getMELAT_ADJTX_STRT_YEAR()), - $(qAdjuvantTxView.getMELAT_ADJTX_END_YEAR()), - $(qAdjuvantTxView.getMELAT_ADJTX_DAYS_DURATION()))) - .from($(qAdjuvantTxView)) - .fetch(); - List radTimelineRecords = darwinQueryFactory.selectDistinct(Projections.constructor(Skcm_mskcc_2015_chantTimelineRadTherapy.class, - $(qRadTherapyView.getMELRT_PTID()), - $(qRadTherapyView.getMELRT_RTTX_TYPE_DESC()), - $(qRadTherapyView.getMELRT_RTTX_ADJ_DESC()), - $(qRadTherapyView.getMELRT_RTTX_STRT_YEAR()), - $(qRadTherapyView.getMELRT_RTTX_END_DT()), - $(qRadTherapyView.getMELAT_RTTX_DAYS_DURATION()))) - .from($(qRadTherapyView)) - .fetch(); - List systemicTimelineRecords = darwinQueryFactory.selectDistinct(Projections.constructor(Skcm_mskcc_2015_chantTimelineSystemicTx.class, - $(qSystemicTxView.getMELST_PTID()), - $(qSystemicTxView.getMELST_SYSTX_TYP_DESC()), - $(qSystemicTxView.getMELST_SYSTX_TYPE_OTH()), - $(qSystemicTxView.getMELST_SYSTX_STRT_YEAR()), - $(qSystemicTxView.getMELST_SYSTX_END_YEAR()), - $(qSystemicTxView.getMELST_SYSTX_DAYS_DURATION()))) - .from($(qSystemicTxView)) - .fetch(); - - darwinRecords.addAll(adjuvantTimelineRecords); - darwinRecords.addAll(radTimelineRecords); - darwinRecords.addAll(systemicTimelineRecords); - - return darwinRecords; - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineWriter.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineWriter.java deleted file mode 100644 index 1583f15ac..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/skcm_mskcc_2015_chanttimeline/Skcm_mskcc_2015_chantTimelineWriter.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.skcm_mskcc_2015_chanttimeline; - -import com.google.common.base.Strings; -import java.io.*; -import java.util.*; -import org.mskcc.cmo.ks.darwin.pipeline.model.Skcm_mskcc_2015_chantTimelineAdjuvantTx; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.*; - -/** - * - * @author heinsz - */ -public class Skcm_mskcc_2015_chantTimelineWriter implements ItemStreamWriter { - - @Value("#{jobParameters[outputDirectory]}") - private String outputDirectory; - - @Value("${darwin.skcm_mskcc_2015_chant_timeline_filename}") - private String filename; - - private int recordsWritten; - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter(); - private File stagingFile; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback(){ - @Override - public void writeHeader(Writer writer) throws IOException { - writer.write(String.join("\t", new Skcm_mskcc_2015_chantTimelineAdjuvantTx().getFieldNames())); - } - }); - stagingFile = new File(outputDirectory, filename); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException { - if (recordsWritten == 0) { - throw new RuntimeException("No records were written to output file: " + stagingFile.getName() + " - exiting..."); - } - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception { - List writeList = new ArrayList<>(); - for (String result : items) { - if (!Strings.isNullOrEmpty(result)) { - writeList.add(result); - recordsWritten++; - } - } - flatFileItemWriter.write(writeList); - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinSampleListUtil.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinSampleListUtil.java deleted file mode 100644 index 9ba91d1f4..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinSampleListUtil.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.util; - -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; - -import org.apache.log4j.Logger; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.*; - -/** - * - * @author Manda Wilson - */ -@Configuration -public class DarwinSampleListUtil { - - private Set filteredMskimpactBrainSpineTimelineSet = new HashSet<>(); - - Logger log = Logger.getLogger(DarwinSampleListUtil.class); - - @Bean - public DarwinSampleListUtil DarwinSampleListUtil() { - return new DarwinSampleListUtil(); - } - - public DarwinSampleListUtil() {} - - /** - * @return the filteredMskimpactBrainSpineTimelineSet - */ - public Set getFilteredMskimpactBrainSpineTimelineSet() { - return filteredMskimpactBrainSpineTimelineSet; - } - - /** - * @param mskimpactBrainSpineTimeline the MskimpactBrainSpineTimeline to add to filteredMskimpactBrainSpineTimelineSet - */ - public void addFilteredMskimpactBrainSpineTimeline(MskimpactBrainSpineTimeline mskimpactBrainSpineTimeline) { - this.filteredMskimpactBrainSpineTimelineSet.add(mskimpactBrainSpineTimeline); - } -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinUtils.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinUtils.java deleted file mode 100644 index 2c446bbe1..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/DarwinUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.util; - -public class DarwinUtils { - - public DarwinUtils() {} - - public String convertWhitespace(String s) { - return s.replaceAll("^[\\t|\\n|\\r]+", "").replaceAll("[\\t|\\n|\\r]+$", "").replaceAll("[\\t|\\n|\\r]+", " "); - } - -} diff --git a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/VitalStatusUtils.java b/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/VitalStatusUtils.java deleted file mode 100644 index ef451b155..000000000 --- a/darwin/src/main/java/org/mskcc/cmo/ks/darwin/pipeline/util/VitalStatusUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin.pipeline.util; - -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactPatientDemographics; - -import java.util.List; -import java.util.ArrayList; - -public class VitalStatusUtils { - - private final static int EIGHTYNINE_YEARS_OLD_IN_DAYS = 32485; - private final static int EIGHTEEN_YEARS_OLD_IN_DAYS = 6570; - - /** - * Generates a row of vital status information. Note, the rules this method follows has been dictated by the vital - * status file format defined by the GENIE clinical data working group consortium: - * https://www.synapse.org/#!Synapse:syn3380222/wiki/495552 (login required) - */ - public static List getVitalStatusResult(DarwinUtils darwinUtils, final MskimpactPatientDemographics darwinPatientDemographics) throws Exception { - List toReturn = new ArrayList<>(); - for (String field : MskimpactPatientDemographics.getVitalStatusFieldNames()) { - Object value = darwinPatientDemographics.getClass().getMethod("get"+field).invoke(darwinPatientDemographics); - if (value == null) { - toReturn.add("NA"); - } - else { - String valueToReturn = darwinUtils.convertWhitespace(value.toString()); - switch(field) { - case "DMP_ID_DEMO": - // nothing to be done - break; - case "LAST_CONTACT_YEAR": - case "PT_DEATH_YEAR": - if (valueToReturn.equals("-1")) { - valueToReturn = "NA"; - } - break; - case "AGE_AT_LAST_CONTACT_YEAR_IN_DAYS": - case "AGE_AT_DATE_OF_DEATH_IN_DAYS": - if (Integer.valueOf(valueToReturn) > EIGHTYNINE_YEARS_OLD_IN_DAYS) { - valueToReturn = ">32485"; - } else if (Integer.valueOf(valueToReturn) < EIGHTEEN_YEARS_OLD_IN_DAYS) { - valueToReturn = "<6570"; - } - break; - case "OS_STATUS": - // OS_STATUS becomes "DEAD" in file - if (valueToReturn.equals("LIVING")) { - valueToReturn = "False"; - } - else if (valueToReturn.equals("DECEASED")) { - valueToReturn = "True"; - } - break; - default: - throw new Exception("Unknown Vital Status field name: " + field); - } - toReturn.add(valueToReturn); - } - } - return toReturn; - } -} diff --git a/darwin/src/main/resources/application.properties.EXAMPLE b/darwin/src/main/resources/application.properties.EXAMPLE deleted file mode 100644 index ddb33676f..000000000 --- a/darwin/src/main/resources/application.properties.EXAMPLE +++ /dev/null @@ -1,64 +0,0 @@ -spring.batch.job.enabled=false - -#server properties -darwin.server= -darwin.port= -darwin.database= -darwin.schema= - -#DB2 Connection -darwin.username= -darwin.password= -darwin.connection_string= - -#Darwin properties -darwin.chunk_size=10 -darwin.demographics_view=DEMOGRAPHICS_V -darwin.latest_activity_view=LATEST_ACTIVITY_V -darwin.pathology_dmp_view=PATHOLOGY_DMP_V -darwin.naaccr.ethnicity_mapping_table=NAACCR_ETHNICITY_CODE_MAPPING -darwin.naaccr.race_mapping_table=NAACCR_RACE_CODE_MAPPING -darwin.naaccr.sex_mapping_table=NAACCR_SEX_CODE_MAPPING -darwin.skcm_mskcc_2015_chant.staging_path_current_view=MEL_STAGING_PATH_CURR_V -darwin.skcm_mskcc_2015_chant.general_view=MEL_GENERAL_V -darwin.skcm_mskcc_2015_chant.primary_view=MEL_PRIMARY_V -darwin.skcm_mskcc_2015_chant.impact_view=MEL_IMPACT_V -darwin.skcm_mskcc_2015_chant.metastat_sites_view=MEL_METASTAT_SITES_V -darwin.skcm_mskcc_2015_chant.systemic_tx=MEL_SYSTEMIC_TX_V -darwin.skcm_mskcc_2015_chant.adjuvant_tx_view=MEL_ADJUVANT_TX_V -darwin.skcm_mskcc_2015_chant.rad_therapy_view=MEL_RAD_THERAPY_V -darwin.demographics_filename=data_clinical_supp_darwin_demographics.txt -darwin.vital_status_filename=darwin/darwin_vital_status.txt -darwin.genie_patient_filename=darwin/darwin_naaccr.txt -darwin.age_filename=darwin/darwin_age.txt -darwin.skcm_mskcc_2015_chant_clinical_sample_filename=data_clinical_sample_darwin_skcm_mskcc_2015_chant.txt -darwin.skcm_mskcc_2015_chant_clinical_patient_filename=data_clinical_patient_darwin_skcm_mskcc_2015_chant.txt -darwin.skcm_mskcc_2015_chant_timeline_filename=data_timeline_skcm_mskcc_2015_chant.txt -darwin.icdo_view=ICDO_V -darwin.timeline_view=DMT_TIMELINE_BRAINSPINE_V -#between prefix and suffix brainspine caisis timeline filenames vary over these types: {diagnosis, specimen, status, surgery, treatment} -darwin.brainspine_timeline_filename_prefix=data_timeline_ -darwin.brainspine_timeline_filename_suffix=_caisis_gbm.txt -darwin.clinical_view=DMT_CLINICAL_BRAINSPINE_V -darwin.brainspine_clinical_filename=data_clinical_supp_caisis_gbm.txt -darwin.medicaltherapy.pharmacy_druglist=IPILIMUMAB;PEMBROLIZUMAB;NIVOLUMAB;ATEZOLIZUMAB # drug names should be in all UPPERCASE -darwin.medicaltherapy_timeline_filename=data_timeline_medical_therapy.txt -darwin.medicaltherapy_clinical_filename=data_clinical_supp_medical_therapy.txt -darwin.medicaltherapy.skip_negative_dosage_records=true - -# For redcap metadata clinical attributes header generation -# Absoulte URL to the redcap web application -redcap_base_url= -redcap_api_url_path= - -# RedCap mapping token for ID_MAPPING - any tables that we want to access need to be in this project -mapping_token= - -# cdd metadata api -cdd_base_url= - -# email properties -email.server=localhost -email.sender= -email.recipient= -email.subject=Failure in Darwin Pipeline diff --git a/darwin/src/main/resources/log4j.properties.EXAMPLE b/darwin/src/main/resources/log4j.properties.EXAMPLE deleted file mode 100644 index 9d220dcfd..000000000 --- a/darwin/src/main/resources/log4j.properties.EXAMPLE +++ /dev/null @@ -1,32 +0,0 @@ -# Change INFO to DEBUG, if you want to see debugging info on underlying libraries we use. -log4j.rootLogger=INFO, a - -# Change INFO to DEBUG, if you want see debugging info on our packages only. -log4j.category.org.mskcc=INFO -#log4j.category.org.springframework=ALL - -#log4j.logger.org.springframework.security=DEBUG -#log4j.logger.org.springframework.integration=DEBUG - - -# Use the JVM option, e.g.: "java -DPORTAL_HOME=/pathto/portal_homedir", -# or - "java -DPORTAL_HOME=$PORTAL_HOME", where PORTAL_HOME is shell (environment) variable. - -## IMPORTANT - THRESHOLD SHOULD NOT BE DEBUG FOR PRODUCTION, CREDENTIALS CAN BE DISPLAYED! - -log4j.appender.a = org.apache.log4j.FileAppender - -log4j.appender.a.File = /path/to/logfile.log - -log4j.appender.a.layout = org.apache.log4j.PatternLayout -log4j.appender.a.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} [%t] %-5p %c - %m%n -log4j.appender.a.append = true - -#log4j.logger.org.hibernate=INFO, a -#log4j.logger.org.hibernate.SQL=DEBUG -#log4j.logger.org.hibernate.type=TRACE -#log4j.logger.org.hibernate.hql.ast.AST=info -#log4j.logger.org.hibernate.tool.hbm2ddl=warn -#log4j.logger.org.hibernate.hql=debug -#log4j.logger.org.hibernate.cache=info -#log4j.logger.org.hibernate.jdbc=debug diff --git a/darwin/src/test/java/org/mskcc/cmo/ks/darwin/DarwinTestConfiguration.java b/darwin/src/test/java/org/mskcc/cmo/ks/darwin/DarwinTestConfiguration.java deleted file mode 100644 index 87eca647a..000000000 --- a/darwin/src/test/java/org/mskcc/cmo/ks/darwin/DarwinTestConfiguration.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin; - -import com.querydsl.core.types.Expression; -import com.querydsl.core.types.Predicate; -import com.querydsl.sql.SQLQuery; -import com.querydsl.sql.SQLQueryFactory; -import java.util.*; -import org.mockito.ArgumentMatchers; -import org.mockito.Mockito; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline.MskimpactTimelineBrainSpineReader; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinSampleListUtil; -import org.springframework.batch.item.ItemStreamReader; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import com.ibm.db2.jcc.DB2SimpleDataSource; -import com.querydsl.sql.SQLQueryFactory; -import com.querydsl.sql.DB2Templates; -import java.sql.SQLException; - -/** - * - * @author Manda Wilson - */ -@Configuration -public class DarwinTestConfiguration { - - @Bean - public DarwinSampleListUtil darwinSampleListUtil() { - return new DarwinSampleListUtil(); - } - - @Bean - public ItemStreamReader mskimpactTimelineBrainSpineReader() { - return new MskimpactTimelineBrainSpineReader(); - } - - @Bean - public SQLQueryFactory darwinQueryFactory() throws SQLException{ - DB2Templates templates = new DB2Templates(); - com.querydsl.sql.Configuration config = new com.querydsl.sql.Configuration(templates); - return new SQLQueryFactory(config, new DB2SimpleDataSource()); - } - -} diff --git a/darwin/src/test/java/org/mskcc/cmo/ks/darwin/MskimpactTimelineBrainSpineReaderTest.java b/darwin/src/test/java/org/mskcc/cmo/ks/darwin/MskimpactTimelineBrainSpineReaderTest.java deleted file mode 100644 index 761bb878c..000000000 --- a/darwin/src/test/java/org/mskcc/cmo/ks/darwin/MskimpactTimelineBrainSpineReaderTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.darwin; - -import java.util.ArrayList; -import java.util.Set; -import org.junit.Assert; -import org.junit.runner.RunWith; -import org.junit.Test; -import org.mskcc.cmo.ks.darwin.pipeline.model.MskimpactBrainSpineTimeline; -import org.mskcc.cmo.ks.darwin.pipeline.mskimpactbrainspinetimeline.MskimpactTimelineBrainSpineReader; -import org.mskcc.cmo.ks.darwin.pipeline.util.DarwinSampleListUtil; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -/** - * - * @author Manda Wilson - */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes=DarwinTestConfiguration.class) -public class MskimpactTimelineBrainSpineReaderTest { - - @Autowired - private MskimpactTimelineBrainSpineReader mskimpactTimelineBrainSpineReader; - - @Autowired - private DarwinSampleListUtil darwinSampleListUtil; - - @Test - public void testReadFiltersNAStartDate() { - mskimpactTimelineBrainSpineReader.openForTestingAndSetDarwinTimelineResults(makeMockMskimpactBrainSpineTimelineResults()); - try { - // confirm DarwinSampleListUtil.filteredMskimpactBrainSpineTimelineSet is empty before we start reading - Set filteredMskimpactBrainSpineTimelineSet = darwinSampleListUtil.getFilteredMskimpactBrainSpineTimelineSet(); - Assert.assertTrue("filteredMskimpactBrainSpineTimelineSet should be empty but contains " + - filteredMskimpactBrainSpineTimelineSet.size() + " records", filteredMskimpactBrainSpineTimelineSet.isEmpty()); - - // read first unfiltered result - MskimpactBrainSpineTimeline actualUnfilteredResult = null; - try { - actualUnfilteredResult = mskimpactTimelineBrainSpineReader.read(); - } catch (Exception e) { - Assert.fail(e.toString()); - } - - // set up expected result - MskimpactBrainSpineTimeline expectedUnfilteredResult = new MskimpactBrainSpineTimeline(); - expectedUnfilteredResult.setDMP_PATIENT_ID_ALL_BRAINSPINETMLN("P-0000003"); - expectedUnfilteredResult.setSTART_DATE("757"); - - // compare expected result and actual result - Assert.assertEquals(expectedUnfilteredResult.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN(), actualUnfilteredResult.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN()); - Assert.assertEquals(expectedUnfilteredResult.getSTART_DATE(), actualUnfilteredResult.getSTART_DATE()); - - // there should be no more records (all others were filtered) - try { - Assert.assertNull("mskimpactTimelineBrainSpineReader should be done reading but found another record", mskimpactTimelineBrainSpineReader.read()); - } catch (Exception e) { - Assert.fail(e.toString()); - } - - // confirm that the filtered records were stored in DarwinSampleListUtil.filteredMskimpactBrainSpineTimelineSet - filteredMskimpactBrainSpineTimelineSet = darwinSampleListUtil.getFilteredMskimpactBrainSpineTimelineSet(); - Assert.assertEquals(2, filteredMskimpactBrainSpineTimelineSet.size()); - boolean foundNAStartDate = false; - boolean foundNullStartDate = false; - for (MskimpactBrainSpineTimeline mskimpactBrainSpineTimeline : filteredMskimpactBrainSpineTimelineSet) { - if (mskimpactBrainSpineTimeline.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN().equals("P-0000001")) { - foundNAStartDate = true; - } else if (mskimpactBrainSpineTimeline.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN().equals("P-0000002")) { - foundNullStartDate = true; - } - Assert.assertTrue(mskimpactBrainSpineTimeline.getDMP_PATIENT_ID_ALL_BRAINSPINETMLN() + - " has unexpected START_DATE '" + mskimpactBrainSpineTimeline.getSTART_DATE() + - "', it should be 'NA'", mskimpactBrainSpineTimeline.getSTART_DATE().equals("NA")); // both "NA" and null should be converted to "NA" in model - } - Assert.assertTrue("Did not find patient initialized with NA start date in darwinSampleListUtil.getFilteredMskimpactBrainSpineTimelineSet()", foundNAStartDate); - Assert.assertTrue("Did not find patient initialized with null start date in darwinSampleListUtil.getFilteredMskimpactBrainSpineTimelineSet()", foundNullStartDate); - - } finally { - mskimpactTimelineBrainSpineReader.close(); - } - } - - private ArrayList makeMockMskimpactBrainSpineTimelineResults() { - ArrayList mskimpactBrainSpineTimelineResults = new ArrayList(); - - MskimpactBrainSpineTimeline mskimpactBrainSpineTimelineNAStartDate = new MskimpactBrainSpineTimeline(); - mskimpactBrainSpineTimelineNAStartDate.setDMP_PATIENT_ID_ALL_BRAINSPINETMLN("P-0000001"); - mskimpactBrainSpineTimelineNAStartDate.setSTART_DATE("NA"); - - MskimpactBrainSpineTimeline mskimpactBrainSpineTimelineNullStartDate = new MskimpactBrainSpineTimeline(); - mskimpactBrainSpineTimelineNullStartDate.setDMP_PATIENT_ID_ALL_BRAINSPINETMLN("P-0000002"); - mskimpactBrainSpineTimelineNullStartDate.setSTART_DATE(null); - - MskimpactBrainSpineTimeline mskimpactBrainSpineTimelineValidStartDate = new MskimpactBrainSpineTimeline(); - mskimpactBrainSpineTimelineValidStartDate.setDMP_PATIENT_ID_ALL_BRAINSPINETMLN("P-0000003"); - mskimpactBrainSpineTimelineValidStartDate.setSTART_DATE("757"); - - mskimpactBrainSpineTimelineResults.add(mskimpactBrainSpineTimelineNAStartDate); // should be filtered by MskimpactTimelineBrainSpineReader.read() - mskimpactBrainSpineTimelineResults.add(mskimpactBrainSpineTimelineNullStartDate); // should be filtered by MskimpactTimelineBrainSpineReader.read() - mskimpactBrainSpineTimelineResults.add(mskimpactBrainSpineTimelineValidStartDate); // should not be filtered by MskimpactTimelineBrainSpineReader.read() - - return mskimpactBrainSpineTimelineResults; - } - -} diff --git a/ddp/ddp_pipeline/pom.xml b/ddp/ddp_pipeline/pom.xml index e5620eda3..631d0ec51 100644 --- a/ddp/ddp_pipeline/pom.xml +++ b/ddp/ddp_pipeline/pom.xml @@ -10,24 +10,24 @@ org.mskcc.cmo.ks.ddp ddp - 0.1.0 + 1.0.0 ddp_source org.mskcc.cmo.ks.ddp.source - 0.1.0 + 1.0.0 org.mskcc.cmo.ks common - 0.1.0 + 1.0.0 org.springframework.batch spring-batch-integration - 4.2.2.RELEASE + ${spring.batch.version} @@ -38,6 +38,7 @@ org.springframework.boot spring-boot-maven-plugin + ${spring.boot.version} org.mskcc.cmo.ks.ddp.DDPPipeline diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/AgeAtSeqDateWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/AgeAtSeqDateWriter.java index 719a320f9..5442b2524 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/AgeAtSeqDateWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/AgeAtSeqDateWriter.java @@ -85,9 +85,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { + public void write(Chunk compositeResults) throws Exception { if (includeAgeAtSeqDate) { - List records = new ArrayList<>(); + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { if (result.getAgeAtSeqDateResults() == null || result.getAgeAtSeqDateResults().isEmpty()) { continue; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/BatchConfiguration.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/BatchConfiguration.java index b447ca156..1cbb6937c 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/BatchConfiguration.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/BatchConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 - 2023 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2018 - 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -37,37 +37,37 @@ import org.mskcc.cmo.ks.ddp.pipeline.model.CompositeResult; import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import javax.sql.DataSource; import org.apache.log4j.Logger; import org.springframework.batch.core.*; import org.springframework.batch.core.configuration.annotation.*; -import org.springframework.batch.core.step.tasklet.Tasklet; -import org.springframework.batch.item.*; -import org.springframework.batch.item.support.CompositeItemWriter; -import org.springframework.beans.factory.annotation.*; -import org.springframework.context.annotation.*; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.batch.integration.async.*; - -import java.util.concurrent.Future; -import java.util.concurrent.Executor; -import javax.sql.DataSource; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.launch.support.SimpleJobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.integration.async.*; +import org.springframework.batch.item.*; +import org.springframework.batch.item.support.CompositeItemWriter; import org.springframework.batch.support.transaction.ResourcelessTransactionManager; +import org.springframework.beans.factory.annotation.*; +import org.springframework.context.annotation.*; import org.springframework.core.io.Resource; import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.jdbc.datasource.init.DataSourceInitializer; import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.transaction.PlatformTransactionManager; /** * * @author ochoaa */ @Configuration -@EnableBatchProcessing @ComponentScan(basePackages = "org.mskcc.cmo.ks.ddp.source") @EnableAsync public class BatchConfiguration { @@ -104,7 +104,6 @@ public ThreadPoolTaskExecutor processorThreadPoolTaskExecutor() { return threadPoolTaskExecutor; } - @Bean @StepScope public ItemProcessor> asyncItemProcessor() { AsyncItemProcessor asyncItemProcessor = new AsyncItemProcessor<>(); @@ -121,11 +120,6 @@ ItemWriter> asyncItemWriter() { } public static final String DDP_COHORT_JOB = "ddpCohortJob"; - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; @Value("${chunk}") private Integer chunkInterval; @@ -133,19 +127,23 @@ ItemWriter> asyncItemWriter() { private final Logger LOG = Logger.getLogger(BatchConfiguration.class); @Bean - public Job ddpCohortJob() { - return jobBuilderFactory.get(DDP_COHORT_JOB) - .start(ddpSeqDateStep()) - .next(ddpStep()) - .next(ddpSortStep()) - .next(ddpEmailStep()) + public Job ddpCohortJob(JobRepository jobRepository, + @Qualifier("ddpSeqDateStep") Step ddpSeqDateStep, + @Qualifier("ddpStep") Step ddpStep, + @Qualifier("ddpSortStep") Step ddpSortStep, + @Qualifier("ddpEmailStep") Step ddpEmailStep) { + return new JobBuilder(DDP_COHORT_JOB, jobRepository) + .start(ddpSeqDateStep) + .next(ddpStep) + .next(ddpSortStep) + .next(ddpEmailStep) .build(); } - @Bean - public Step ddpStep() { - return stepBuilderFactory.get("ddpStep") - .> chunk(chunkInterval) + @Bean(name = "ddpStep") + public Step ddpStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("ddpStep", jobRepository) + .> chunk(chunkInterval, transactionManager) .reader(ddpReader()) .processor(asyncItemProcessor()) .writer(asyncItemWriter()) @@ -246,40 +244,46 @@ public CompositeItemWriter ddpCompositeWriter() { return writer; } - @Bean - public Step ddpSeqDateStep() { - return stepBuilderFactory.get("ddpSeqDateStep") - .tasklet(ddpSeqDateTasklet()) + @Bean(name = "ddpSeqDateStep") + public Step ddpSeqDateStep(JobRepository jobRepository, + @Qualifier("ddpSeqDateTasklet") Tasklet ddpSeqDateTasklet, + PlatformTransactionManager transactionManager) { + return new StepBuilder("ddpSeqDateStep", jobRepository) + .tasklet(ddpSeqDateTasklet, transactionManager) .build(); } - @Bean + @Bean(name = "ddpSeqDateTasklet") @StepScope public Tasklet ddpSeqDateTasklet() { return new DDPSeqDateTasklet(); } - @Bean - public Step ddpSortStep() { - return stepBuilderFactory.get("ddpSortStep") - .tasklet(ddpSortTasklet()) + @Bean(name = "ddpSortStep") + public Step ddpSortStep(JobRepository jobRepository, + @Qualifier("ddpSortTasklet") Tasklet ddpSortTasklet, + PlatformTransactionManager transactionManager) { + return new StepBuilder("ddpSortStep", jobRepository) + .tasklet(ddpSortTasklet, transactionManager) .build(); } - @Bean + @Bean(name = "ddpSortTasklet") @StepScope public Tasklet ddpSortTasklet() { return new DDPSortTasklet(); } - @Bean - public Step ddpEmailStep() { - return stepBuilderFactory.get("ddpEmailStep") - .tasklet(ddpEmailTasklet()) + @Bean(name = "ddpEmailStep") + public Step ddpEmailStep(JobRepository jobRepository, + @Qualifier("ddpEmailTasklet") Tasklet ddpEmailTasklet, + PlatformTransactionManager transactionManager) { + return new StepBuilder("ddpEmailStep", jobRepository) + .tasklet(ddpEmailTasklet, transactionManager) .build(); } - @Bean + @Bean(name = "ddpEmailTasklet") @StepScope public Tasklet ddpEmailTasklet() { return new DDPEmailTasklet(); diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/ClinicalWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/ClinicalWriter.java index c2251b144..c491b7254 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/ClinicalWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/ClinicalWriter.java @@ -87,8 +87,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { - List records = new ArrayList<>(); + public void write(Chunk compositeResults) throws Exception { + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { records.add(result.getClinicalResult()); } diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/DDPReader.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/DDPReader.java index 38c18d692..09ee93c9b 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/DDPReader.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/DDPReader.java @@ -39,9 +39,9 @@ import org.mskcc.cmo.ks.ddp.pipeline.util.DDPPatientListUtil; import com.google.common.base.Strings; +import jakarta.annotation.Resource; import java.io.*; import java.util.*; -import javax.annotation.Resource; import org.apache.log4j.Logger; import org.springframework.batch.item.*; import org.springframework.beans.factory.annotation.*; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppNaaccrMappingsWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppNaaccrMappingsWriter.java index 6d9079aac..4d823c609 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppNaaccrMappingsWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppNaaccrMappingsWriter.java @@ -92,9 +92,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { + public void write(Chunk compositeResults) throws Exception { if (DDPUtils.isMskimpactCohort(cohortName) || DDPUtils.isHemepactCohort(cohortName) || DDPUtils.isMskaccessCohort(cohortName)) { - List records = new ArrayList<>(); + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { if (Strings.isNullOrEmpty(result.getSuppNaccrMappingsResult())) { continue; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppVitalStatusWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppVitalStatusWriter.java index 954431565..059255197 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppVitalStatusWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/SuppVitalStatusWriter.java @@ -92,9 +92,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { + public void write(Chunk compositeResults) throws Exception { if (DDPUtils.isMskimpactCohort(cohortName) || DDPUtils.isHemepactCohort(cohortName) || DDPUtils.isMskaccessCohort(cohortName)) { - List records = new ArrayList<>(); + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { if (Strings.isNullOrEmpty(result.getSuppVitalStatusResult())) { continue; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineChemoWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineChemoWriter.java index 70ddeadff..66c6bab0d 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineChemoWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineChemoWriter.java @@ -85,9 +85,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { + public void write(Chunk compositeResults) throws Exception { if (includeChemotherapy) { - List records = new ArrayList<>(); + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { if (result.getTimelineChemoResults() == null || result.getTimelineChemoResults().isEmpty()) { continue; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineRadiationWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineRadiationWriter.java index 877c72149..19c6656bf 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineRadiationWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineRadiationWriter.java @@ -85,9 +85,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { + public void write(Chunk compositeResults) throws Exception { if (includeRadiation) { - List records = new ArrayList<>(); + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { if (result.getTimelineRadiationResults() == null || result.getTimelineRadiationResults().isEmpty()) { continue; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineSurgeryWriter.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineSurgeryWriter.java index 8167a9f5e..ee655fc85 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineSurgeryWriter.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/TimelineSurgeryWriter.java @@ -85,9 +85,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List compositeResults) throws Exception { + public void write(Chunk compositeResults) throws Exception { if (includeSurgery) { - List records = new ArrayList<>(); + Chunk records = new Chunk<>(); for (CompositeResult result : compositeResults) { if (result.getTimelineSurgeryResults()== null || result.getTimelineSurgeryResults().isEmpty()) { continue; diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/AgeAtSeqDateRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/AgeAtSeqDateRecord.java index 0786add97..19c37bf0d 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/AgeAtSeqDateRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/AgeAtSeqDateRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2023, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -36,7 +36,6 @@ import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -97,11 +96,6 @@ public void setAGE_AT_SEQ_REPORTED_YEARS(String AGE_AT_SEQ_REPORTED_YEARS) { this.AGE_AT_SEQ_REPORTED_YEARS = AGE_AT_SEQ_REPORTED_YEARS; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/ClinicalRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/ClinicalRecord.java index 594f87ec2..af90cc88e 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/ClinicalRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/ClinicalRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2018, 2019, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -37,7 +37,6 @@ import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -242,11 +241,6 @@ public void setSURGERY(String SURGERY) { this.SURGERY = SURGERY; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppNaaccrMappingsRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppNaaccrMappingsRecord.java index 91f98f762..9622790ec 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppNaaccrMappingsRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppNaaccrMappingsRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2019, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -37,7 +37,6 @@ import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -163,11 +162,6 @@ public void setBIRTH_YEAR(String BIRTH_YEAR) { this.BIRTH_YEAR = BIRTH_YEAR; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppVitalStatusRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppVitalStatusRecord.java index c4e20c6b2..81ed32f06 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppVitalStatusRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/SuppVitalStatusRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2019, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -38,7 +38,6 @@ import com.google.common.base.Strings; import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -147,11 +146,6 @@ public void setDEAD(String DEAD) { this.DEAD = DEAD; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineChemoRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineChemoRecord.java index cb9a7a114..fbd9464d7 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineChemoRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineChemoRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2018, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -37,7 +37,6 @@ import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -162,11 +161,6 @@ public void setAGENT(String AGENT) { this.AGENT = AGENT; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineRadiationRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineRadiationRecord.java index b99ce0ff9..46a7a27c1 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineRadiationRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineRadiationRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2018, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -38,7 +38,6 @@ import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -245,11 +244,6 @@ public void setREF_POINT_SITE(String REF_POINT_SITE) { this.REF_POINT_SITE = REF_POINT_SITE; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineSurgeryRecord.java b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineSurgeryRecord.java index 784163119..8d2eb78f0 100644 --- a/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineSurgeryRecord.java +++ b/ddp/ddp_pipeline/src/main/java/org/mskcc/cmo/ks/ddp/pipeline/model/TimelineSurgeryRecord.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2018, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -37,7 +37,6 @@ import java.text.ParseException; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; /** * @@ -114,11 +113,6 @@ public void setSURGERY_DETAILS(String SURGERY_DETAILS) { this.SURGERY_DETAILS = SURGERY_DETAILS; } - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - /** * Returns field names as list of strings. * diff --git a/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/pipeline/util/DDPUtilsTest.java b/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/pipeline/util/DDPUtilsTest.java index 5c8dd5810..e0fe8eb03 100644 --- a/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/pipeline/util/DDPUtilsTest.java +++ b/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/pipeline/util/DDPUtilsTest.java @@ -40,8 +40,8 @@ import org.mskcc.cmo.ks.ddp.source.model.PatientDemographics; import org.mskcc.cmo.ks.ddp.source.model.PatientDiagnosis; +import jakarta.annotation.Resource; import java.util.*; -import javax.annotation.Resource; import org.junit.Assert; import org.junit.runner.RunWith; import org.junit.Test; diff --git a/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/source/internal/DDPRepositoryTest.java b/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/source/internal/DDPRepositoryTest.java index 731f94522..e023867ce 100644 --- a/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/source/internal/DDPRepositoryTest.java +++ b/ddp/ddp_pipeline/src/test/java/org/mskcc/cmo/ks/ddp/source/internal/DDPRepositoryTest.java @@ -38,7 +38,7 @@ import org.junit.*; import org.junit.runner.RunWith; import java.util.*; -import javax.annotation.Resource; +import jakarta.annotation.Resource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; diff --git a/ddp/ddp_source/pom.xml b/ddp/ddp_source/pom.xml index 7431c43a9..d933c7622 100644 --- a/ddp/ddp_source/pom.xml +++ b/ddp/ddp_source/pom.xml @@ -9,6 +9,6 @@ org.mskcc.cmo.ks.ddp ddp - 0.1.0 + 1.0.0 diff --git a/ddp/pom.xml b/ddp/pom.xml index 6db5c2884..86c060746 100644 --- a/ddp/pom.xml +++ b/ddp/pom.xml @@ -10,7 +10,7 @@ org.mskcc.cmo.ks master - 0.1.0 + 1.0.0 ddp_pipeline diff --git a/docs/DMP-Updates-Workflow.md b/docs/DMP-Updates-Workflow.md index bbb47c312..b013579fe 100644 --- a/docs/DMP-Updates-Workflow.md +++ b/docs/DMP-Updates-Workflow.md @@ -239,60 +239,6 @@ fi # TODO: move other pre-import/data-fetch steps here (i.e exporting raw files from redcap) ``` -**DARWIN CAISIS GBM CLINICAL AND TIMELINE FETCH** - -``` -printTimeStampedDataProcessingStepMessage "MSKIMPACT data processing" -# fetch darwin caisis data -printTimeStampedDataProcessingStepMessage "Darwin CAISIS fetch for mskimpact" -$JAVA_BINARY $JAVA_DARWIN_FETCHER_ARGS -s mskimpact -d $MSK_IMPACT_DATA_HOME -c -if [ $? -gt 0 ] ; then - cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard - sendPreImportFailureMessageMskPipelineLogsSlack "MSKIMPACT Darwin CAISIS Fetch" -else - FETCH_DARWIN_CAISIS_FAIL=0 - echo "committing darwin caisis data" - cd $MSK_IMPACT_DATA_HOME ; $GIT_BINARY add ./* ; $GIT_BINARY commit -m "Latest MSKIMPACT Dataset: Darwin CAISIS" -fi -``` - -> $JAVA_BINARY $JAVA_DARWIN_FETCHER_ARGS -s mskimpact -d $MSK_IMPACT_DATA_HOME -c - -********************************************************************** - -***[GITHUB | DMP Repository State: 1](#github-dmp-repository-state-1)*** - -*Untracked files* -* CAISIS Clinical and Timeline files: - * `$MSK_IMPACT_DATA_HOME/data_clinical_supp_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_imaging_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_specimen_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_status_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_surgery_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_treatment_caisis_gbm.txt` -* Supp Date Added files: - * `$MSK_IMPACT_DATA_HOME/data_clinical_mskimpact_supp_date_cbioportal_added.txt` - * `$MSK_HEMEPACT_DATA_HOME/data_clinical_hemepact_data_clinical_supp_date.txt` - * `$MSK_RAINDANCE_DATA_HOME/data_clinical_mskraindance_data_clinical_supp_date.txt` - * `$MSK_ARCHER_UNFILTERED_DATA_HOME/data_clinical_mskarcher_data_clinical_supp_date.txt` - * `$MSK_ACCESS_DATA_HOME/data_clinical_mskaccess_data_clinical_supp_date.txt` -* CVR Clinical files: - * `$MSK_IMPACT_DATA_HOME/data_clinical_mskimpact_data_clinical_cvr.txt` - * `$MSK_HEMEPACT_DATA_HOME/data_clinical_hemepact_data_clinical.txt` - * `$MSK_RAINDANCE_DATA_HOME/data_clinical_mskraindance_data_clinical.txt` - * `$MSK_ARCHER_UNFILTERED_DATA_HOME/data_clinical_mskarcher_data_clinical.txt` - * `$MSK_ACCESS_DATA_HOME/data_clinical_mskaccess_data_clinical.txt` - -*FAILURE:* -> cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard - -**--> Untracked files from [GITHUB | DMP Repository State: 1](#github-dmp-repository-state-1) are not affected by `reset`** - -*SUCCESS* -> cd $MSK_IMPACT_DATA_HOME ; $GIT_BINARY add ./* ; $GIT_BINARY commit -m "Latest MSKIMPACT Dataset: Darwin CAISIS" - -**--> Untracked files from [GITHUB | DMP Repository State: 1](#github-dmp-repository-state-1) in `$MSK_IMPACT_DATA_HOME` and CAISIS clinical and timeline output files are tracked and added during this commit.** - ********************************************************************** **CVR CLINICAL AND GENOMIC FETCH** @@ -1527,19 +1473,6 @@ if [ $PERFORM_CRDB_FETCH -gt 0 ] && [ $FETCH_CRDB_IMPACT_FAIL -eq 0 ] ; then fi ``` -* **DARWIN CAISIS** - -``` -# imports mskimpact darwin data into redcap -if [ $FETCH_DARWIN_CAISIS_FAIL -eq 0 ] ; then - import_mskimpact_darwin_caisis_to_redcap - if [ $? -gt 0 ] ; then - IMPORT_STATUS_IMPACT=1 - sendPreImportFailureMessageMskPipelineLogsSlack "MSKIMPACT Darwin CAISIS Redcap Import" - fi -fi -``` - * **DDP** ``` @@ -1760,15 +1693,9 @@ $GIT_BINARY commit -m "Raw clinical and timeline file cleanup: MSKIMPACT, HEMEPA * `$MSK_IMPACT_DATA_HOME/data_clinical_ddp_pediatrics.txt` * `$MSK_IMPACT_DATA_HOME/data_clinical_mskimpact_data_clinical_cvr.txt` * `$MSK_IMPACT_DATA_HOME/data_clinical_mskimpact_supp_date_cbioportal_added.txt` - * `$MSK_IMPACT_DATA_HOME/data_clinical_supp_caisis_gbm.txt` * `$MSK_IMPACT_DATA_HOME/data_timeline_ddp_chemotherapy.txt` * `$MSK_IMPACT_DATA_HOME/data_timeline_ddp_radiation.txt` * `$MSK_IMPACT_DATA_HOME/data_timeline_ddp_surgery.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_imaging_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_specimen_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_status_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_surgery_caisis_gbm.txt` - * `$MSK_IMPACT_DATA_HOME/data_timeline_treatment_caisis_gbm.txt` * HEMEPACT: * `$MSK_HEMEPACT_DATA_HOME/data_clinical_ddp.txt` * `$MSK_HEMEPACT_DATA_HOME/data_clinical_hemepact_data_clinical.txt` diff --git a/gdd/pom.xml b/gdd/pom.xml deleted file mode 100644 index ddd44dd0a..000000000 --- a/gdd/pom.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - 4.0.0 - MSKCC CMO GDD Pipeline - Genome Directed Diagnosis pipeline - gdd - 0.1.0 - - - org.mskcc.cmo.ks - master - 0.1.0 - - - - - org.springframework - spring-web - ${spring.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - commons-lang - commons-lang - 2.4 - - - commons-cli - commons-cli - 1.3 - - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - org.mskcc.cmo.ks.gdd.GDDPipeline - - - - - - diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/GDDPipeline.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/GDDPipeline.java deleted file mode 100644 index 3d35f1078..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/GDDPipeline.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.cbioportal.cmo.pipelines; - -import org.cbioportal.cmo.pipelines.gdd.BatchConfiguration; - -import org.apache.commons.cli.*; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.batch.core.*; -import org.springframework.batch.core.launch.JobLauncher; - -/** - * @author Benjamin Gross - */ -@SpringBootApplication -public class GDDPipeline -{ - - private static Options getOptions(String[] args) - { - Options options = new Options(); - options.addOption("h", "help", false, "shows this help document and quits.") - .addOption("maf", "maf", true, "MAF file") - .addOption("cna", "cna", true, "CNA file") - .addOption("seg", "seg", true, "SEG file") - .addOption("sv", "sv", true, "SV file") - .addOption("cl", "cl", true, "CLINICAL file") - .addOption("stage", "staging", true, "Staging filename"); - - return options; - } - - private static void help(Options options, int exitStatus) - { - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.printHelp("GDDPipeline", options); - System.exit(exitStatus); - } - - private static void launchJob(String[] args, String maf, String cna, - String seg, String sv, - String clinical, String stagingFile) throws Exception - { - SpringApplication app = new SpringApplication(GDDPipeline.class); - ConfigurableApplicationContext ctx= app.run(args); - JobLauncher jobLauncher = ctx.getBean(JobLauncher.class); - - Job gddJob = ctx.getBean(BatchConfiguration.GDD_JOB, Job.class); - JobParameters jobParameters = new JobParametersBuilder() - .addString("maf", maf) - .addString("cna", cna) - .addString("seg", seg) - .addString("sv", sv) - .addString("clinical", clinical) - .addString("stagingFile", stagingFile) - .toJobParameters(); - JobExecution jobExecution = jobLauncher.run(gddJob, jobParameters); - } - - public static void main(String[] args) throws Exception - { - Options options = GDDPipeline.getOptions(args); - CommandLineParser parser = new DefaultParser(); - CommandLine commandLine = parser.parse(options, args); - if (commandLine.hasOption("h") || - !commandLine.hasOption("maf") || - !commandLine.hasOption("cna") || - !commandLine.hasOption("seg") || - !commandLine.hasOption("sv") || - !commandLine.hasOption("cl") || - !commandLine.hasOption("stage")) { - help(options, 0); - } - if (args.length < 1) { - String maf = "/home/ochoaa/data/gdd/data_mutations_extended.txt"; - String cna = "/home/ochoaa/data/gdd/data_CNA.txt"; - String seg = "/home/ochoaa/data/gdd/data_seg.seg"; - String sv = "/home/ochoaa/data/gdd/data_SV.txt"; - String cl = "/home/ochoaa/data/gdd/data_clinical.txt"; - String stage = "/home/ochoaa/data/gdd/Results/data_gdd.txt"; - launchJob(args, maf,cna, seg, sv, cl, stage); - } - else { - launchJob(args, commandLine.getOptionValue("maf"), - commandLine.getOptionValue("cna"), commandLine.getOptionValue("seg"), - commandLine.getOptionValue("sv"), commandLine.getOptionValue("cl"), - commandLine.getOptionValue("stage")); - } - } -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/BatchConfiguration.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/BatchConfiguration.java deleted file mode 100644 index cc1af1b69..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/BatchConfiguration.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2016, 2023 Memorial Sloan Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.cbioportal.cmo.pipelines.gdd; - -import org.cbioportal.cmo.pipelines.gdd.model.GDDResult; -import org.springframework.batch.core.*; -import org.springframework.batch.core.configuration.annotation.*; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.*; - -/** - * @author Benjamin Gross - */ -@Configuration -@EnableBatchProcessing -public class BatchConfiguration { - public static final String GDD_JOB = "gddJob"; - - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; - - @Bean - public Job gddJob() { - return jobBuilderFactory.get(GDD_JOB) - .start(step()) - .build(); - } - - @Bean - public Step step() { - return stepBuilderFactory.get("step") - . chunk(10) - .reader(reader()) - .processor(processor()) - .writer(writer()) - .build(); - } - - @Bean - @StepScope - public ItemStreamReader reader() { - return new GDDClassifierReader(); - } - - @Bean - public GDDClassifierProcessor processor() { - return new GDDClassifierProcessor(); - } - - @Bean - @StepScope - public ItemStreamWriter writer() { - return new GDDResultWriter(); - } - -} - - diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierProcessor.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierProcessor.java deleted file mode 100644 index 993c835cb..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierProcessor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.cbioportal.cmo.pipelines.gdd; - -import org.cbioportal.cmo.pipelines.gdd.model.GDDResult; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.batch.item.ItemProcessor; - -/** - * @author Benjamin Gross - */ -public class GDDClassifierProcessor implements ItemProcessor -{ - ObjectMapper mapper = new ObjectMapper(); - - @Override - public String process(final GDDResult gddResult) throws Exception - { - return gddResult.getSampleId() + "\t" + mapper.writeValueAsString(gddResult); - } -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierReader.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierReader.java deleted file mode 100644 index 14d6f090c..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDClassifierReader.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.cbioportal.cmo.pipelines.gdd; - -import org.cbioportal.cmo.pipelines.gdd.model.*; - -import org.springframework.http.*; -import org.springframework.batch.item.*; -import org.springframework.web.client.RestTemplate; -import org.springframework.core.io.FileSystemResource; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.beans.factory.annotation.Value; - -import java.util.*; -/** - * @author Benjamin Gross - */ -public class GDDClassifierReader implements ItemStreamReader -{ - @Value("${gdd.url}") - private String gddURL; - - @Value("#{jobParameters[maf]}") - private String maf; - - @Value("#{jobParameters[cna]}") - private String cna; - - @Value("#{jobParameters[seg]}") - private String seg; - - @Value("#{jobParameters[sv]}") - private String sv; - - @Value("#{jobParameters[clinical]}") - private String clinical; - - private List results; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException - { - RestTemplate restTemplate = new RestTemplate(); - HttpEntity> requestEntity = getRequestEntity(); - ResponseEntity responseEntity = - restTemplate.exchange(gddURL, HttpMethod.POST, requestEntity, GDDClassifier.class); - this.results = responseEntity.getBody().getResult(); - } - - private HttpEntity> getRequestEntity() - { - LinkedMultiValueMap map = new LinkedMultiValueMap<>(); - map.add("maf", new FileSystemResource(maf)); - map.add("cna", new FileSystemResource(cna)); - map.add("seg", new FileSystemResource(seg)); - map.add("sv", new FileSystemResource(sv)); - map.add("clinical", new FileSystemResource(clinical)); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - return new HttpEntity>(map, headers); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException {} - - @Override - public GDDResult read() throws Exception - { - // necessary until bug that generates duplicate results in classifier script "generate_feature_table.R" gets fixed - if (!results.isEmpty() && results.get(0).getSampleId() != null) { - return results.remove(0); - } - return null; - } -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDResultWriter.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDResultWriter.java deleted file mode 100644 index 571b6b6e6..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/GDDResultWriter.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.cbioportal.cmo.pipelines.gdd; - -import org.springframework.core.io.*; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.transform.PassThroughLineAggregator; -import org.springframework.beans.factory.annotation.Value; -import java.io.*; -import java.util.*; - -/** - * @author Benjamin Gross - */ -public class GDDResultWriter implements ItemStreamWriter -{ - @Value("#{jobParameters[stagingFile]}") - private String stagingFile; - - //private Resource resource; - private List writeList = new ArrayList(); - private FlatFileItemWriter flatFileItemWriter = new FlatFileItemWriter(); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException - { - PassThroughLineAggregator aggr = new PassThroughLineAggregator(); - flatFileItemWriter.setLineAggregator(aggr); - flatFileItemWriter.setHeaderCallback(new FlatFileHeaderCallback() { - @Override - public void writeHeader(Writer writer) throws IOException { - writer.write("SAMPLE_ID\tCLASSIFICATION"); - } - }); - flatFileItemWriter.setResource(new FileSystemResource(stagingFile)); - flatFileItemWriter.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException - { - flatFileItemWriter.close(); - } - - @Override - public void write(List items) throws Exception - { - writeList.clear(); - List writeList = new ArrayList(); - for (String result : items) { - writeList.add(result); - } - flatFileItemWriter.write(writeList); - } -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassification.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassification.java deleted file mode 100644 index d03d75b28..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassification.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.cbioportal.cmo.pipelines.gdd.model; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import org.apache.commons.lang.builder.ToStringBuilder; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("org.jsonschema2pojo") -public class GDDClassification { - -@JsonIgnore -private Map additionalProperties = new HashMap(); - -@Override -public String toString() { -return ToStringBuilder.reflectionToString(this); -} - -@JsonAnyGetter -public Map getAdditionalProperties() { -return this.additionalProperties; -} - -@JsonAnySetter -public void setAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -} - -public GDDClassification withAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -return this; -} - -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassifier.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassifier.java deleted file mode 100644 index 7b02cf79c..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDClassifier.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.cbioportal.cmo.pipelines.gdd.model; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import org.apache.commons.lang.builder.ToStringBuilder; - - -/** -* Genome Directed Diagnosis Schema -*

-* Output format description for GDD web service -* -*/ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("org.jsonschema2pojo") -@JsonPropertyOrder({ -"version", -"result" -}) -public class GDDClassifier { - -/** -* -* (Required) -* -*/ -@JsonProperty("version") -private String version; -/** -* -* (Required) -* -*/ -@JsonProperty("result") -private List result = new ArrayList(); -@JsonIgnore -private Map additionalProperties = new HashMap(); - -/** -* No args constructor for use in serialization -* -*/ -public GDDClassifier() { -} - -/** -* -* @param result -* @param version -*/ -public GDDClassifier(String version, List result) { -this.version = version; -this.result = result; -} - -/** -* -* (Required) -* -* @return -* The version -*/ -@JsonProperty("version") -public String getVersion() { -return version; -} - -/** -* -* (Required) -* -* @param version -* The version -*/ -@JsonProperty("version") -public void setVersion(String version) { -this.version = version; -} - -public GDDClassifier withVersion(String version) { -this.version = version; -return this; -} - -/** -* -* (Required) -* -* @return -* The result -*/ -@JsonProperty("result") -public List getResult() { -return result; -} - -/** -* -* (Required) -* -* @param result -* The result -*/ -@JsonProperty("result") -public void setResult(List result) { -this.result = result; -} - -public GDDClassifier withResult(List result) { -this.result = result; -return this; -} - -@Override -public String toString() { -return ToStringBuilder.reflectionToString(this); -} - -@JsonAnyGetter -public Map getAdditionalProperties() { -return this.additionalProperties; -} - -@JsonAnySetter -public void setAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -} - -public GDDClassifier withAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -return this; -} - -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDEvidence.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDEvidence.java deleted file mode 100644 index b4e8ca4ca..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDEvidence.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.cbioportal.cmo.pipelines.gdd.model; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import org.apache.commons.lang.builder.ToStringBuilder; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("org.jsonschema2pojo") -public class GDDEvidence { - -@JsonIgnore -private Map additionalProperties = new HashMap(); - -@Override -public String toString() { -return ToStringBuilder.reflectionToString(this); -} - -@JsonAnyGetter -public Map getAdditionalProperties() { -return this.additionalProperties; -} - -@JsonAnySetter -public void setAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -} - -public GDDEvidence withAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -return this; -} - -} diff --git a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDResult.java b/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDResult.java deleted file mode 100644 index 1643f13bd..000000000 --- a/gdd/src/main/java/org/mskcc/cmo/ks/gdd/pipeline/model/GDDResult.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -package org.cbioportal.cmo.pipelines.gdd.model; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import org.apache.commons.lang.builder.ToStringBuilder; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("org.jsonschema2pojo") -@JsonPropertyOrder({ -"sample_id", -"cancer_type", -"classification", -"evidence" -}) -public class GDDResult { - -@JsonProperty("sample_id") -private String sampleId; -@JsonProperty("cancer_type") -private String cancerType; -@JsonProperty("classification") -private List classification = new ArrayList(); -@JsonProperty("evidence") -private GDDEvidence evidence; -@JsonIgnore -private Map additionalProperties = new HashMap(); - -/** -* No args constructor for use in serialization -* -*/ -public GDDResult() { -} - -/** -* -* @param cancerType -* @param evidence -* @param classification -* @param sampleId -*/ -public GDDResult(String sampleId, String cancerType, List classification, GDDEvidence evidence) { -this.sampleId = sampleId; -this.cancerType = cancerType; -this.classification = classification; -this.evidence = evidence; -} - -/** -* -* @return -* The sampleId -*/ -@JsonProperty("sample_id") -public String getSampleId() { -return sampleId; -} - -/** -* -* @param sampleId -* The sample_id -*/ -@JsonProperty("sample_id") -public void setSampleId(String sampleId) { -this.sampleId = sampleId; -} - -public GDDResult withSampleId(String sampleId) { -this.sampleId = sampleId; -return this; -} - -/** -* -* @return -* The cancerType -*/ -@JsonProperty("cancer_type") -public String getCancerType() { -return cancerType; -} - -/** -* -* @param cancerType -* The cancer_type -*/ -@JsonProperty("cancer_type") -public void setCancerType(String cancerType) { -this.cancerType = cancerType; -} - -public GDDResult withCancerType(String cancerType) { -this.cancerType = cancerType; -return this; -} - -/** -* -* @return -* The classification -*/ -@JsonProperty("classification") -public List getClassification() { -return classification; -} - -/** -* -* @param classification -* The classification -*/ -@JsonProperty("classification") -public void setClassification(List classification) { -this.classification = classification; -} - -public GDDResult withClassification(List classification) { -this.classification = classification; -return this; -} - -/** -* -* @return -* The evidence -*/ -@JsonProperty("evidence") -public GDDEvidence getEvidence() { -return evidence; -} - -/** -* -* @param evidence -* The evidence -*/ -@JsonProperty("evidence") -public void setEvidence(GDDEvidence evidence) { -this.evidence = evidence; -} - -public GDDResult withEvidence(GDDEvidence evidence) { -this.evidence = evidence; -return this; -} - -@Override -public String toString() { -return ToStringBuilder.reflectionToString(this); -} - -@JsonAnyGetter -public Map getAdditionalProperties() { -return this.additionalProperties; -} - -@JsonAnySetter -public void setAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -} - -public GDDResult withAdditionalProperty(String name, Object value) { -this.additionalProperties.put(name, value); -return this; -} - -} diff --git a/gdd/src/main/resources/application.properties.EXAMPLE b/gdd/src/main/resources/application.properties.EXAMPLE deleted file mode 100644 index 20bb4bf9c..000000000 --- a/gdd/src/main/resources/application.properties.EXAMPLE +++ /dev/null @@ -1,2 +0,0 @@ -spring.batch.job.enabled= -gdd.url= \ No newline at end of file diff --git a/gdd/src/main/resources/gdd-sample.json b/gdd/src/main/resources/gdd-sample.json deleted file mode 100644 index 78049d3a1..000000000 --- a/gdd/src/main/resources/gdd-sample.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "version": "1.0", - "result": [ - { - "sample_id": "1", - "cancer_type": "Lung.Adenocarcinoma", - "classification": [ - {"Breast.Carcinoma": "0.10"}, - {"Lung.Adenocarcinoma": "0.90"} - ], - "evidence": { - "Breast.Carcinoma": [{ - "feature": "1TERTp", - "MI": "1.9e-02", - "VarImp": "1.3e-01" - }, { - "feature": "1CDKN1A", - "MI": "1.6e-02", - "VarImp": "1.7e-02" - }, { - "feature": "1RB1", - "MI": "1.8e-03", - "VarImp": "1.8e-02" - }], - "Lung.Adenocarcinoma": [{ - "feature": "1KMT2D", - "MI": "1.2e-03", - "VarImp": "1.5e-02" - }, { - "feature": "1SMARCA4", - "MI": "1.1e-03", - "VarImp": "1.6e-03" - }, { - "feature": "1ERBB3", - "MI": "1.6e-03", - "VarImp": "1.8e-02" - }] - } - }, - { - "sample_id": "2", - "cancer_type": "GBM", - "classification": [ - {"PROSTATE": "0.20"}, - {"GBM": "0.80"} - ], - "evidence": { - "PROSTATE": [{ - "feature": "2TERTp", - "MI": "2.9e-02", - "VarImp": "2.3e-01" - }, { - "feature": "2CDKN1A", - "MI": "2.6e-02", - "VarImp": "2.7e-02" - }, { - "feature": "2RB1", - "MI": "2.8e-03", - "VarImp": "2.8e-02" - }], - "GBM": [{ - "feature": "2KMT2D", - "MI": "2.2e-03", - "VarImp": "2.5e-02" - }, { - "feature": "2SMARCA4", - "MI": "2.1e-03", - "VarImp": "2.6e-03" - }, { - "feature": "2ERBB3", - "MI": "2.6e-03", - "VarImp": "2.8e-02" - }] - } - }] -} diff --git a/gdd/src/main/resources/gdd-schema.json b/gdd/src/main/resources/gdd-schema.json deleted file mode 100644 index 3b120ddb5..000000000 --- a/gdd/src/main/resources/gdd-schema.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "$schema": "http://json-schema.org/schema#", - "title": "Genome Directed Diagnosis Schema", - "description": "Output format description for GDD web service", - "type": "object", - "required": ["version", "result"], - "properties": { - "version": { - "type": "string" - }, - "result": { - "type": "array", - "items": { - "type": "object", - "properties": { - "sample_id": { - "type": "string" - }, - "cancer_type": { - "type": "string" - }, - "classification": { - "$ref": "#/definitions/classification" - }, - "evidence": { - "$ref": "#/definitions/evidence" - } - } - } - } - }, - "definitions": { - "classification": { - "type": "array", - "items": { - "type": "object", - "patternProperties": { - "^[a-zA-Z\\.]+$": { - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "patternProperties": { - "^[a-zA-Z\\.]+$": { - "type": "array", - "items": { - "type": "object", - "properties": { - "feature": { - "type": "string" - }, - "MI": { - "type": "string" - }, - "VarImp": { - "type": "string" - } - } - } - } - } - } - } -} diff --git a/gene/pom.xml b/gene/pom.xml deleted file mode 100644 index d4d8edc35..000000000 --- a/gene/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - 4.0.0 - MSKCC CMO Gene Data Pipeline - Gene Data Pipeline - gene - 0.1.0 - - jar - - - org.mskcc.cmo.ks - master - 0.1.0 - - - - - org.springframework - spring-web - ${spring.version} - - - commons-lang - commons-lang - 2.4 - - - commons-cli - commons-cli - 1.3 - - - commons-collections - commons-collections - 3.2.1 - jar - - - - org.springframework - spring-jdbc - jar - - - commons-dbcp - commons-dbcp - 1.4 - jar - - - mysql - mysql-connector-java - - - com.google.guava - guava - 19.0 - jar - - - commons-io - commons-io - 2.7 - jar - - - - - gene_data_updater - - - - org.springframework.boot - spring-boot-maven-plugin - - org.mskcc.cmo.ks.GeneDataPipeline - - - - - diff --git a/gene/src/main/java/org/mskcc/cmo/ks/GeneDataPipeline.java b/gene/src/main/java/org/mskcc/cmo/ks/GeneDataPipeline.java deleted file mode 100644 index b799de0be..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/GeneDataPipeline.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks; - -import org.mskcc.cmo.ks.gene.config.BatchConfiguration; - -import org.apache.commons.cli.*; -import org.apache.log4j.Logger; -import org.springframework.batch.core.*; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.ConfigurableApplicationContext; - -/** - * - * @author ochoaa - */ - -@SpringBootApplication -public class GeneDataPipeline { - - private final static Logger LOG = Logger.getLogger(GeneDataPipeline.class); - - private static void launchGeneDataJob(String[] args, String geneDataFileName, String geneLengthDataFileName, String notificationFileName) throws Exception { - SpringApplication app = new SpringApplication(GeneDataPipeline.class); - ConfigurableApplicationContext ctx = app.run(args); - JobLauncher jobLauncher = ctx.getBean(JobLauncher.class); - - JobParameters jobParameters = new JobParametersBuilder() - .addString("geneDataFileName", geneDataFileName) - .addString("geneLengthDataFileName", geneLengthDataFileName) - .addString("notificationFileName", notificationFileName) - .toJobParameters(); - Job geneDataJob = ctx.getBean(BatchConfiguration.GENE_DATA_JOB, Job.class); - JobExecution jobExecution = jobLauncher.run(geneDataJob, jobParameters); - if (!jobExecution.getExitStatus().equals(ExitStatus.COMPLETED)) { - LOG.error("GeneDataPipeline job failed with exit status: " + jobExecution.getExitStatus()); - System.exit(1); - } - } - - private static Options getOptions(String[] args) { - Options options = new Options(); - options.addOption("h", "help", false, "shows this help document and quits.") - .addOption("d", "gene-data-filename", true, "The gene data filename (Homo_sapiens.gene_info)") - .addOption("l", "gene-length-filename", true, "The gene length data filename (GFF file)") - .addOption("n", "notification-filename", true, "The notification filename to write results from update [optional]"); - return options; - } - - private static void help(Options options, int exitStatus) { - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.printHelp("GeneDataPipeline", options); - System.exit(exitStatus); - } - - public static void main(String[] args) throws Exception { - Options options = GeneDataPipeline.getOptions(args); - CommandLineParser parser = new DefaultParser(); - CommandLine commandLine = parser.parse(options, args); - if (commandLine.hasOption("h") || !commandLine.hasOption("d") || !commandLine.hasOption("l")) { - help(options, 0); - } - launchGeneDataJob(args, commandLine.getOptionValue("d"), commandLine.getOptionValue("l"), commandLine.hasOption("n") ? commandLine.getOptionValue("n") : null); - } -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataListener.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataListener.java deleted file mode 100644 index 4bef57205..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataListener.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene; - -import java.util.*; -import java.io.*; -import org.apache.log4j.Logger; -import org.mskcc.cmo.ks.gene.model.Gene; -import org.springframework.batch.core.*; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.beans.factory.annotation.Value; - -/** - * - * @author ochoaa - */ -public class GeneDataListener implements StepExecutionListener { - - @Value("${DATABASE_NAME}") - private String DATABASE_NAME; - - private static final Logger LOG = Logger.getLogger(GeneDataListener.class); - - @Override - public void beforeStep(StepExecution stepExecution) {} - - @Override - public ExitStatus afterStep(StepExecution stepExecution) { - String notificationFileName = (String) stepExecution.getJobParameters().getString("notificationFileName"); - - int genesAdded = (int) stepExecution.getExecutionContext().getInt("genesAdded", 0); - int genesUpdated = (int) stepExecution.getExecutionContext().getInt("genesUpdated", 0); - int geneAliasesAdded = (int) stepExecution.getExecutionContext().getInt("geneAliasesAdded", 0); - - StringBuilder builder = new StringBuilder(); - if ((genesAdded + genesUpdated + geneAliasesAdded) == 0) { - builder.append("Nothing to update in database: ") - .append(DATABASE_NAME); - } - else { - builder.append("Gene update results for database: ") - .append(DATABASE_NAME) - .append("\n\tNew genes added to database: ") - .append(genesAdded) - .append("\n\tTotal genes updated in database: ") - .append(genesUpdated) - .append("\n\tNew aliases added to database: ") - .append(geneAliasesAdded); - } - builder.append("\n\n"); - LOG.info(builder.toString()); - - // report ambiguous genes from database not found in NCBI human gene info file - Map ambiguousGenesToReport = (Map) stepExecution.getExecutionContext().get("ambiguousGenesToReport"); - - if (!ambiguousGenesToReport.isEmpty()) { - LOG.info("Found " + ambiguousGenesToReport.size() + " ambiguous genes to report."); - builder.append("Ambiguous genes report: ") - .append(ambiguousGenesToReport.size()) - .append(" ambiguous genes found in ") - .append(DATABASE_NAME) - .append("\n\nAmbiguous genes were found in the database that no longer exist in the NCBI human gene info file. ") - .append("These genes may have been discontinued and replaced with new entrez gene IDs and/or hugo gene symbols. ") - .append("Please verify on NCBI (https://www.ncbi.nlm.nih.gov/gene/).\n\n"); - - builder.append("Hugo_Gene_Symbol\tEntrez_Gene_Id\n"); - for (Integer entrezGeneId : ambiguousGenesToReport.keySet()) { - builder.append(ambiguousGenesToReport.get(entrezGeneId).getHugoGeneSymbol()) - .append("\t") - .append(entrezGeneId) - .append("\n"); - } - } - else { - LOG.info("No ambiguous or discontinued genes to report."); - builder.append("No ambiguous or likely discontinued genes to report."); - } - - if (notificationFileName != null) { - File notificationFile = new File(notificationFileName); - try { - org.apache.commons.io.FileUtils.writeStringToFile(notificationFile, builder.toString()); - } catch (IOException ex) { - LOG.error("Error writing results to notification file!"); - throw new ItemStreamException(ex); - } - } - - return ExitStatus.COMPLETED; - } - -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataProcessor.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataProcessor.java deleted file mode 100644 index b7994de5f..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataProcessor.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene; - -import org.mskcc.cmo.ks.gene.model.Gene; -import org.mskcc.cmo.ks.gene.model.GeneAlias; - -import java.util.*; -import javax.annotation.Resource; -import org.springframework.batch.item.ItemProcessor; - -/** - * - * @author ochoaa - */ -public class GeneDataProcessor implements ItemProcessor { - - @Resource(name="entrezGeneIdMap") - private Map entrezGeneIdMap; - - @Override - public Gene process(Gene geneRecord) throws Exception { - // check entrezGeneIdMap if gene exists - if does not exist then set genetic entity id to -1 and return gene record - // otherwise, set genetic entity id to the existing gene genetic entity id from database - Gene existingGene = entrezGeneIdMap.get(geneRecord.getEntrezGeneId()); - if (existingGene == null) { - geneRecord.setGeneticEntityId(-1); - return geneRecord; - } - else { - geneRecord.setGeneticEntityId(existingGene.getGeneticEntityId()); - } - // if gene needs updating then return updated gene record, otherwise return null to filter out record from GeneDataWriter - if (geneNeedsUpdating(geneRecord, existingGene)) { - return geneRecord; - } - return null; - } - - /** - * Determines whether gene record needs updating in the database. - * 1. Compare hugo symbols - if not equal then hugo symbol from database is added as an alias to current gene record - * 2. Compare gene aliases - if not equal then remove aliases from current gene record that already exist in database. - * Only new aliases should remain in set for current gene record. - * 3. Compare the following: - * - if size of current set of gene record aliases is greater than 1 - * - if gene type doesn't match what's in the database - * - if gene cytoband doesn't match what's in the database - * - if gene length doesn't match what's in the database - * - * @param geneRecord - * @param existingGene - * @return - */ - private boolean geneNeedsUpdating(Gene geneRecord, Gene existingGene) { - boolean needsUpdating = false; - // update gene aliases for current gene record if hugo symbols don't match - if (!geneRecord.getHugoGeneSymbol().equalsIgnoreCase(existingGene.getHugoGeneSymbol())) { - // assuming that we do not have alias in `gene_alias` table for current entrez id and existing gene hugo symbol - geneRecord.getAliases().add(new GeneAlias(geneRecord.getEntrezGeneId(), existingGene.getHugoGeneSymbol())); - needsUpdating = true; - } - // make sure that we remove any GeneAlias objects from current gene record that already exist in database - for (GeneAlias alias : existingGene.getAliases()) { - if (geneRecord.getAliases().contains(alias)) { - geneRecord.getAliases().remove(alias); - } - } - - // set update to true if any of these conditions are met - if (geneRecord.getAliases().size() > 1 || !geneRecord.getType().equals(existingGene.getType()) || - !geneRecord.getCytoband().equals(existingGene.getCytoband()) || !Objects.equals(geneRecord.getLength(), existingGene.getLength())) { - needsUpdating = true; - } - return needsUpdating; - } -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataReader.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataReader.java deleted file mode 100644 index 0726422e7..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataReader.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene; - -import org.mskcc.cmo.ks.gene.model.Gene; -import org.mskcc.cmo.ks.gene.model.GeneAlias; - -import java.io.*; -import java.util.*; -import java.util.regex.*; -import javax.annotation.Resource; -import org.apache.log4j.Logger; -import org.springframework.batch.item.*; -import org.springframework.batch.item.file.*; -import org.springframework.batch.item.file.mapping.*; -import org.springframework.batch.item.file.transform.*; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.FileSystemResource; - -/** - * - * @author ochoaa - */ -public class GeneDataReader implements ItemStreamReader { - - private final String GENE_ID_COLUMN = "GeneID"; - private final String SYMBOL_COLUMN = "Symbol"; - private final String MAIN_SYMBOL_COLUMN = "Symbol_from_nomenclature_authority"; // default to this value if not '-' - private final String SYNONYMS_COLUMN = "Synonyms"; // also locus_tag is used for aliases - private final String LOCUS_TAG_COLUMN = "LocusTag"; - private final String TYPE_OF_GENE_COLUMN = "type_of_gene"; - private final String CYTOBAND_COLUMN = "map_location"; - private final String MIR_GENE_NAME_PATTERN = "MIR"; - private final String PROTEIN_CODING_GENE_TYPE = "protein-coding"; - - private final Integer GENETIC_REGION_INDEX = 2; - private final Integer GENETIC_INFORMATION_INDEX = 8; - private final Integer GENE_START_POS_INDEX = 3; - private final Integer GENE_END_POS_INDEX = 4; - - - private final Pattern ENTREZ_ID_PATTERN = Pattern.compile(".*GeneID:([0-9]*).*"); - - @Value("#{jobParameters[geneDataFileName]}") - private String geneDataFileName; - - @Value("#{jobParameters[geneLengthDataFileName]}") - private String geneLengthDataFileName; - - @Resource(name="ambiguousGenesMap") - private Map ambiguousGenesMap; - - private Map geneLengthDataMap = new HashMap<>(); - private List geneRecords = new ArrayList(); - - private final static Logger LOG = Logger.getLogger(GeneDataReader.class); - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - try { - this.geneLengthDataMap = loadGeneLengthData(); - } catch (Exception ex) { - LOG.error("Error loading gene length data from: " + geneLengthDataFileName); - throw new ItemStreamException(ex); - } - try { - this.geneRecords = loadHumanGeneData(); - } catch (Exception ex) { - LOG.error("Error loading data from: " + geneDataFileName); - throw new ItemStreamException(ex); - } - executionContext.put("ambiguousGenesToReport", ambiguousGenesMap); - } - - /** - * Loads human gene data from Homo_sapiens.gene_info. - * - * @return List - * @throws FileNotFoundException - * @throws ItemStreamException - */ - private List loadHumanGeneData() throws Exception { - LOG.info("Loading data from: " + geneDataFileName); - File geneInfoFile = new File(geneDataFileName); - // init line mapper and field tokenizer - final DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(DelimitedLineTokenizer.DELIMITER_TAB); - DefaultLineMapper lineMapper = new DefaultLineMapper(); - lineMapper.setLineTokenizer(tokenizer); - lineMapper.setFieldSetMapper((FieldSet fs) -> fs.getProperties()); - - // set reader resources and load data - FlatFileItemReader reader = new FlatFileItemReader(); - reader.setResource(new FileSystemResource(geneInfoFile)); - reader.setLineMapper(lineMapper); - reader.setLinesToSkip(1); - reader.setSkippedLinesCallback((String line) -> { - tokenizer.setNames(line.split(DelimitedLineTokenizer.DELIMITER_TAB)); - }); - reader.open(new ExecutionContext()); - - List geneFileRecords = new ArrayList(); - Properties record; - while ((record = reader.read()) != null) { - Integer entrezGeneId = Integer.valueOf(record.getProperty(GENE_ID_COLUMN)); - String hugoGeneSymbol = record.getProperty(MAIN_SYMBOL_COLUMN).equals("-") ? record.getProperty(SYMBOL_COLUMN) : record.getProperty(MAIN_SYMBOL_COLUMN); - String type = record.getProperty(TYPE_OF_GENE_COLUMN); - String cytoband = record.getProperty(CYTOBAND_COLUMN); - - // get synonyms from 'synonyms' column and 'locus tag' column, then construct list of GeneAlias objects - List synonyms = new ArrayList(); - if (!record.getProperty(SYNONYMS_COLUMN).equals("-")) { - synonyms.addAll(Arrays.asList(record.getProperty(SYNONYMS_COLUMN).split("[\\|\\;]"))); - } - if (!record.getProperty(LOCUS_TAG_COLUMN).equals("-")) { - synonyms.add(record.getProperty(LOCUS_TAG_COLUMN)); - } - // skip record if gene is a microRNA - if (isMirGeneRecord(hugoGeneSymbol, synonyms, type)) { - continue; - } - Set aliases = new HashSet<>(); - for (String alias : synonyms) { - aliases.add(new GeneAlias(entrezGeneId, alias.trim().toUpperCase())); - } - geneFileRecords.add(new Gene(entrezGeneId, hugoGeneSymbol.toUpperCase(), type, cytoband, geneLengthDataMap.get(entrezGeneId), aliases)); - // remove gene record from ambiguous gene map if exists (indicating that it's a valid record in NCBI gene file) - // remaining ambiguous genes in this map will be reported to data curation team as likely discontinued records - // that should be removed from the database - ambiguousGenesMap.remove(entrezGeneId); - } - reader.close(); - - return geneFileRecords; - } - - /** - * Checks whether record is a microRNA based on primary hugo symbol and gene synonyms. - * @param hugoGeneSymbol - * @param synonyms - * @return - */ - private boolean isMirGeneRecord(String hugoGeneSymbol, List synonyms, String type) { - // don't filter out records where gene type is "protein-coding", - if (type.equalsIgnoreCase(PROTEIN_CODING_GENE_TYPE)) { - return Boolean.FALSE; - } - boolean isMirGene = (hugoGeneSymbol.toUpperCase().startsWith(MIR_GENE_NAME_PATTERN)); - // check synonyms if hugo gene symbol doesn't match MIR_GENE_NAME_PATTERN - if (!isMirGene) { - for (String synonym : synonyms) { - if (synonym.toUpperCase().startsWith(MIR_GENE_NAME_PATTERN)) { - isMirGene = Boolean.TRUE; - break; - } - } - } - return isMirGene; - } - - /** - * Load and calculate gene length information from GFF file. - * This file does not contain a header - the following is the relevant information to extract: - * col 3: genetic region (i.e., exon, CDS, gene, mRNA, transcript, - * col 4: gene start position - * col 5: gene end position - * col 9: ';'-delimited genetic information, only the 'GeneID' (entrez gene id) will be extracted - * ex: - * ID=id1;Parent=rna0;Dbxref=GeneID:100287102,Genbank:NR_046018.2,HGNC:HGNC:37102;gbkey=misc_RNA;gene=DDX11L1;product=DEAD/H-box helicase 11 like 1;transcript_id=NR_046018.2 - * - * The gene length is calculated by calculating the difference between the max gene position found and min gene position found. - * - * @return Map - * @throws FileNotFoundException - * @throws IOException - */ - private Map loadGeneLengthData() throws Exception { - LOG.info("Loading data from: " + geneLengthDataFileName); - Map> genePositionData = new HashMap<>(); - BufferedReader buf = new BufferedReader(new FileReader(new File(geneLengthDataFileName))); - String line; - while ((line = buf.readLine()) != null) { - if (line.startsWith("#")) { - continue; - } - String[] parts = line.split("\t"); - if (!parts[GENETIC_REGION_INDEX].equalsIgnoreCase("exon") && !parts[GENETIC_REGION_INDEX].equalsIgnoreCase("cds")) { - continue; - } - // extract entrez gene id from genetic information - Matcher matcher = ENTREZ_ID_PATTERN.matcher(parts[GENETIC_INFORMATION_INDEX]); - Integer entrezGeneId = matcher.find() ? Integer.valueOf(matcher.group(1)) : null; - - // make sure entrez gene id was extracted successfully, throw exception if value couldn't be parsed - if (entrezGeneId == null) { - LOG.error("Could not extract entrez gene id from row: \n\t" + line); - throw new ItemStreamException("Check logic for extracting entrez gene id from gene length data file."); - } - // update gene position map with exon/cds position data - Set positions = genePositionData.getOrDefault(entrezGeneId, new HashSet<>()); - positions.addAll(Arrays.asList(new Integer[]{Integer.valueOf(parts[GENE_START_POS_INDEX]), Integer.valueOf(parts[GENE_END_POS_INDEX])})); - genePositionData.put(entrezGeneId, positions); - } - buf.close(); - - // get the gene length by calculating the difference between the max gene position and min gene position - Map geneLengthDataMap = new HashMap<>(); - for (Integer entrezGeneId : genePositionData.keySet()) { - Integer length = Collections.max(genePositionData.get(entrezGeneId)) - Collections.min(genePositionData.get(entrezGeneId)) + 1; - geneLengthDataMap.put(entrezGeneId, length); - } - return geneLengthDataMap; - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void close() throws ItemStreamException {} - - @Override - public Gene read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { - if (!geneRecords.isEmpty()) { - return geneRecords.remove(0); - } - return null; - } - -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataWriter.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataWriter.java deleted file mode 100644 index 982860ea8..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/GeneDataWriter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene; - -import java.util.ArrayList; -import java.util.List; -import org.mskcc.cmo.ks.gene.jdbc.internal.DaoGeneJdbcImpl; -import org.mskcc.cmo.ks.gene.model.Gene; -import org.mskcc.cmo.ks.gene.model.GeneAlias; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.batch.item.ItemStreamWriter; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * - * @author ochoaa - */ -public class GeneDataWriter implements ItemStreamWriter { - - @Autowired - private DaoGeneJdbcImpl daoGeneJdbcImpl; - - private int genesAdded; - private int genesUpdated; - private int geneAliasesAdded; - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException {} - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - executionContext.put("genesAdded", genesAdded); - executionContext.put("genesUpdated", genesUpdated); - executionContext.put("geneAliasesAdded", geneAliasesAdded); - } - - @Override - public void close() throws ItemStreamException {} - - @Override - public void write(List list) throws Exception { - List newGenes = new ArrayList(); - List genesToUpdate = new ArrayList(); - List aliasesToUpdate = new ArrayList(); - - for (Gene gene : list) { - if (gene.getGeneticEntityId() == -1) { - gene.setGeneticEntityId(daoGeneJdbcImpl.addGeneGeneticEntity()); - newGenes.add(gene); - } - else { - genesToUpdate.add(gene); - } - aliasesToUpdate.addAll(gene.getAliases()); - } - - this.genesAdded += daoGeneJdbcImpl.batchInsertGene(newGenes); - this.genesUpdated += daoGeneJdbcImpl.batchUpdateGene(genesToUpdate); - this.geneAliasesAdded += daoGeneJdbcImpl.batchInsertGeneAlias(aliasesToUpdate); - } - -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/config/BatchConfiguration.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/config/BatchConfiguration.java deleted file mode 100644 index 4d972a827..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/config/BatchConfiguration.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.config; - -import java.net.MalformedURLException; -import javax.sql.DataSource; -import org.mskcc.cmo.ks.gene.*; -import org.mskcc.cmo.ks.gene.model.Gene; - -import org.springframework.batch.core.*; -import org.springframework.batch.core.configuration.annotation.*; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.support.SimpleJobLauncher; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; -import org.springframework.batch.item.*; -import org.springframework.batch.support.transaction.ResourcelessTransactionManager; -import org.springframework.beans.factory.annotation.*; -import org.springframework.context.annotation.*; -import org.springframework.core.io.Resource; -import org.springframework.jdbc.datasource.DriverManagerDataSource; -import org.springframework.jdbc.datasource.init.DataSourceInitializer; -import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; -import org.springframework.transaction.PlatformTransactionManager; - -/** - * - * @author ochoaa - */ -@Configuration -@EnableBatchProcessing -public class BatchConfiguration { - - public static final String GENE_DATA_JOB = "geneDataJob"; - - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; - - @Value("${chunk.interval}") - private int chunkInterval; - - @Bean - public Job geneDataJob() { - return jobBuilderFactory.get(GENE_DATA_JOB) - .start(geneDataStep()) - .build(); - } - - @Bean - public Step geneDataStep() { - return stepBuilderFactory.get("geneDataStep") - . chunk(chunkInterval) - .reader(geneDataReader()) - .processor(geneDataProcessor()) - .writer(geneDataWriter()) - .listener(geneDataListener()) - .build(); - } - - @Bean - @StepScope - public ItemStreamReader geneDataReader() { - return new GeneDataReader(); - } - - @Bean - @StepScope - public GeneDataProcessor geneDataProcessor() { - return new GeneDataProcessor(); - } - - @Bean - @StepScope - public ItemStreamWriter geneDataWriter() { - return new GeneDataWriter(); - } - - @Bean - public StepExecutionListener geneDataListener() { - return new GeneDataListener(); - } - - // general spring batch configuration - @Value("org/springframework/batch/core/schema-drop-sqlite.sql") - private Resource dropRepositoryTables; - - @Value("org/springframework/batch/core/schema-sqlite.sql") - private Resource dataRepositorySchema; - - /** - * Spring Batch datasource. - * @return DataSource - */ - @Bean - public DataSource dataSource() { - DriverManagerDataSource dataSource = new DriverManagerDataSource(); - dataSource.setDriverClassName("org.sqlite.JDBC"); - dataSource.setUrl("jdbc:sqlite:repository.sqlite"); - return dataSource; - } - - /** - * Spring Batch datasource initializer. - * @param dataSource - * @return DataSourceInitializer - * @throws MalformedURLException - */ - @Bean - public DataSourceInitializer dataSourceInitializer(DataSource dataSource) throws MalformedURLException { - ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator(); - databasePopulator.addScript(dropRepositoryTables); - databasePopulator.addScript(dataRepositorySchema); - databasePopulator.setIgnoreFailedDrops(true); - - DataSourceInitializer initializer = new DataSourceInitializer(); - initializer.setDataSource(dataSource); - initializer.setDatabasePopulator(databasePopulator); - return initializer; - } - - /** - * Spring Batch job repository. - * @return JobRepository - * @throws Exception - */ - private JobRepository getJobRepository() throws Exception { - JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean(); - factory.setDataSource(dataSource()); - factory.setTransactionManager(getTransactionManager()); - factory.afterPropertiesSet(); - return (JobRepository) factory.getObject(); - } - - /** - * Spring Batch transaction manager. - * @return PlatformTransactionManager - */ - private PlatformTransactionManager getTransactionManager() { - return new ResourcelessTransactionManager(); - } - - /** - * Spring Batch job launcher. - * @return JobLauncher - * @throws Exception - */ - public JobLauncher getJobLauncher() throws Exception { - SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); - jobLauncher.setJobRepository(getJobRepository()); - jobLauncher.afterPropertiesSet(); - return jobLauncher; - } -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/config/JdbcConfiguration.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/config/JdbcConfiguration.java deleted file mode 100644 index fdd552c7b..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/config/JdbcConfiguration.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2017-2019 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.config; - -import org.springframework.context.annotation.Configuration; - -import java.sql.SQLException; -import javax.sql.DataSource; -import org.apache.commons.dbcp.BasicDataSource; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.*; -import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; -import org.springframework.transaction.*; - -/** - * - * @author ochoaa - */ -@Configuration -public class JdbcConfiguration { - - @Value("${db.user}") - private String dbUser; - - @Value("${db.password}") - private String dbPassword; - - @Value("${db.driver}") - private String dbDriver; - - @Value("${db.url}") - private String dbUrl; - - @Value("${db.use_ssl:false}") - private Boolean dbUseSsl; - - @Value("${DATABASE_NAME}") - private String DATABASE_NAME; - - public DataSource mainDataSource() throws SQLException { - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUsername(dbUser); - dataSource.setPassword(dbPassword); - dataSource.setDriverClassName(dbDriver); - dataSource.setUrl(dbUrl + DATABASE_NAME + "?useSSL=" + String.valueOf(dbUseSsl)); - System.out.println(dataSource.getUrl()); - dataSource.setDefaultTransactionIsolation(TransactionDefinition.ISOLATION_READ_UNCOMMITTED); - return dataSource; - } - - /** - * Bean that holds the named parameter jdbc template. - * - * @return NamedParameterJdbcTemplate - * @throws SQLException - */ - @Bean(name="namedParameterJdbcTemplate") - public NamedParameterJdbcTemplate namedParameterJdbcTemplate() throws SQLException { - return new NamedParameterJdbcTemplate(mainDataSource()); - } -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/DaoGeneJdbc.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/DaoGeneJdbc.java deleted file mode 100644 index 46baeaad2..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/DaoGeneJdbc.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.jdbc; - -import org.mskcc.cmo.ks.gene.model.Gene; -import org.mskcc.cmo.ks.gene.model.GeneAlias; - -import java.util.*; - -/** - * - * @author ochoaa - */ -public interface DaoGeneJdbc { - Gene addGene(Gene gene); - void addGeneAliases(Gene gene); - Integer addGeneGeneticEntity(); - void updateGene(Gene gene); - Gene getGene(Integer entrezGeneId); - Set getGeneAliases(Integer entrezGeneId); - List getAllGenes(); - List getAllAmbiguousHugoGeneSymbols(); - List getAllAmbiguousGenes(); - void deleteGeneGeneticEntity(Integer geneticEntityId); - Integer batchInsertGene(List genes) throws Exception; - Integer batchUpdateGene(List genes) throws Exception; - Integer batchInsertGeneAlias(List geneAliases) throws Exception; -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/internal/DaoGeneJdbcImpl.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/internal/DaoGeneJdbcImpl.java deleted file mode 100644 index e158b9e13..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/jdbc/internal/DaoGeneJdbcImpl.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.jdbc.internal; - -import org.mskcc.cmo.ks.gene.jdbc.DaoGeneJdbc; -import org.mskcc.cmo.ks.gene.model.Gene; -import org.mskcc.cmo.ks.gene.model.GeneAlias; - -import java.sql.*; -import java.util.*; -import javax.annotation.Resource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cglib.beans.BeanMap; -import org.springframework.dao.*; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.namedparam.*; -import org.springframework.jdbc.support.*; -import org.springframework.stereotype.Repository; - -/** - * - * @author ochoaa - */ -@Repository -public class DaoGeneJdbcImpl implements DaoGeneJdbc { - - @Resource(name="namedParameterJdbcTemplate") - private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; - - @Autowired - public DaoGeneJdbcImpl(NamedParameterJdbcTemplate namedParameterJdbcTemplate) { - this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; - } - - @Override - public Gene addGene(Gene gene) throws DataAccessException { - String SQL = "INSERT INTO gene " + - "(entrez_gene_id, hugo_gene_symbol, genetic_entity_id, type, cytoband, length) " + - "VALUES(:entrez_gene_id, :hugo_gene_symbol, :genetic_entity_id, :type, :cytoband, :length)"; - MapSqlParameterSource paramMap = new MapSqlParameterSource(); - paramMap.addValue("entrez_gene_id", gene.getEntrezGeneId()); - paramMap.addValue("hugo_gene_symbol", gene.getHugoGeneSymbol().toUpperCase()); - paramMap.addValue("genetic_entity_id", gene.getGeneticEntityId()); - paramMap.addValue("type", gene.getType()); - paramMap.addValue("cytoband", gene.getCytoband()); - paramMap.addValue("length", gene.getLength()); - namedParameterJdbcTemplate.update(SQL, paramMap); - - // add gene aliases if not empty - if (!gene.getAliases().isEmpty()) { - addGeneAliases(gene); - } - - return gene; - } - - @Override - public void addGeneAliases(Gene gene) throws DataAccessException { - String SQL = "INSERT INTO gene_alias " + - "(entrez_gene_id, gene_alias) " + - "VALUES(:entrez_gene_id, :gene_alias)"; - MapSqlParameterSource paramMap = new MapSqlParameterSource(); - paramMap.addValue("entrez_gene_id", gene.getEntrezGeneId()); - - for (GeneAlias alias : gene.getAliases()) { - paramMap.addValue("gene_alias", alias.getGeneAlias()); - namedParameterJdbcTemplate.update(SQL, paramMap); - } - } - - @Override - public Integer addGeneGeneticEntity() throws DataAccessException { - String SQL = "INSERT INTO genetic_entity (entity_type) VALUES(:val)"; - SqlParameterSource paramMap = new MapSqlParameterSource("val", "GENE"); - KeyHolder keyHolder = new GeneratedKeyHolder(); - namedParameterJdbcTemplate.update(SQL, paramMap, keyHolder, new String[] {"ID"}); - Integer geneticEntityId =Integer.valueOf( keyHolder.getKeyList().get(0).get("GENERATED_KEY").toString()); - return geneticEntityId; - } - - @Override - public void updateGene(Gene gene) throws DataAccessException { - String SQL = "UPDATE gene " + - "SET hugo_gene_symbol = :hugo_gene_symbol, type = :type, cytoband = :cytoband, length = :length " + - "WHERE entrez_gene_id = :entrez_gene_id"; - MapSqlParameterSource paramMap = new MapSqlParameterSource(); - paramMap.addValue("hugo_gene_symbol", gene.getHugoGeneSymbol()); - paramMap.addValue("type", gene.getType()); - paramMap.addValue("cytoband", gene.getCytoband()); - paramMap.addValue("length", gene.getLength()); - paramMap.addValue("entrez_gene_id", gene.getEntrezGeneId()); - namedParameterJdbcTemplate.update(SQL, paramMap); - - // only new gene aliases should be in list, shouldn't have duplicates after executing insert into 'gene_alias' table - if (!gene.getAliases().isEmpty()) { - addGeneAliases(gene); - } - } - - @Override - public Gene getGene(Integer entrezGeneId) { - String SQL = "SELECT " + - "gene.entrez_gene_id AS 'gene.entrezGeneId', " + - "gene.hugo_gene_symbol AS 'gene.hugoGeneSymbol', " + - "gene.genetic_entity_id as 'gene.geneticEntityId', " + - "gene.type AS 'gene.type', " + - "gene.cytoband AS 'gene.cytoband', " + - "gene.length AS 'gene.length' " + - "FROM gene " + - "WHERE gene.entrez_gene_id = :entrez_gene_id "; - SqlParameterSource paramMap = new MapSqlParameterSource("entrez_gene_id", entrezGeneId); - - Gene gene = null; - try { - gene = (Gene) namedParameterJdbcTemplate.queryForObject(SQL, paramMap, geneRowMapper()); - } - catch (EmptyResultDataAccessException ex) {} - - return gene; - } - - @Override - public Set getGeneAliases(Integer entrezGeneId) throws DataAccessException { - String SQL = "SELECT DISTINCT gene_alias from gene_alias WHERE entrez_gene_id = :entrez_gene_id"; - SqlParameterSource paramMap = new MapSqlParameterSource("entrez_gene_id", entrezGeneId); - List aliases = namedParameterJdbcTemplate.queryForList(SQL, paramMap, String.class); - - Set geneAliases = new HashSet<>(); - for (String alias : aliases) { - if (alias.contains(";")) { - continue; - } - geneAliases.add(new GeneAlias(entrezGeneId, alias.toUpperCase())); - } - return geneAliases; - } - - @Override - public List getAllGenes() { - String SQL = "SELECT " + - "gene.entrez_gene_id AS 'gene.entrezGeneId', " + - "gene.hugo_gene_symbol AS 'gene.hugoGeneSymbol', " + - "gene.genetic_entity_id AS 'gene.geneticEntityId', " + - "gene.type AS 'gene.type', " + - "gene.cytoband AS 'gene.cytoband', " + - "gene.length AS 'gene.length' " + - "FROM gene " + - "GROUP BY gene.entrez_gene_id"; - - List genes = new ArrayList(); - try { - genes = (List) namedParameterJdbcTemplate.query(SQL, geneRowMapper()); - } - catch (EmptyResultDataAccessException ex) {} - - return genes; - } - - @Override - public List getAllAmbiguousHugoGeneSymbols() { - String SQL = "SELECT DISTINCT gene.hugo_gene_symbol FROM gene GROUP BY gene.hugo_gene_symbol HAVING COUNT(*) > 1"; - List ambiguousGeneSymbols = namedParameterJdbcTemplate.query(SQL, (ResultSet rs, int i) -> rs.getString(1)); - return ambiguousGeneSymbols; - } - - @Override - public List getAllAmbiguousGenes() { - String SQL = "SELECT " + - "gene.entrez_gene_id AS 'gene.entrezGeneId', " + - "gene.hugo_gene_symbol AS 'gene.hugoGeneSymbol', " + - "gene.genetic_entity_id AS 'gene.geneticEntityId', " + - "gene.type AS 'gene.type', " + - "gene.cytoband AS 'gene.cytoband', " + - "gene.length AS 'gene.length' " + - "FROM gene " + - "WHERE gene.hugo_gene_symbol IN (:ambiguous_genes_list)"; - SqlParameterSource paramMap = new MapSqlParameterSource("ambiguous_genes_list", getAllAmbiguousHugoGeneSymbols()); - - List ambiguousGenes = new ArrayList(); - try { - ambiguousGenes = (List) namedParameterJdbcTemplate.query(SQL, paramMap, geneRowMapper()); - } - catch (EmptyResultDataAccessException ex) {} - - return ambiguousGenes; - } - - @Override - public void deleteGeneGeneticEntity(Integer geneticEntityId) throws DataAccessException { - String SQL = "DELETE FROM genetic_entity WHERE id = :genetic_entity_id"; - SqlParameterSource paramMap = new MapSqlParameterSource("genetic_entity_id", geneticEntityId); - namedParameterJdbcTemplate.update(SQL, paramMap); - } - - @Override - public Integer batchInsertGene(List genes) throws Exception { - String SQL = "INSERT INTO gene " + - "(entrez_gene_id, hugo_gene_symbol, genetic_entity_id, type, cytoband, length) " + - "VALUES(:entrezGeneId, :hugoGeneSymbol, :geneticEntityId, :type, :cytoband, :length)"; - int[] rows = namedParameterJdbcTemplate.batchUpdate(SQL, SqlParameterSourceUtils.createBatch(genes.toArray(new Gene[genes.size()]))); - return rows != null ? Arrays.stream(rows).sum() : 0; - } - - @Override - public Integer batchUpdateGene(List genes) throws Exception { - String SQL = "UPDATE gene " + - "SET hugo_gene_symbol = :hugoGeneSymbol, type = :type, cytoband = :cytoband, length = :length " + - "WHERE entrez_gene_id = :entrezGeneId"; - int[] rows = namedParameterJdbcTemplate.batchUpdate(SQL, SqlParameterSourceUtils.createBatch(genes.toArray(new Gene[genes.size()]))); - return rows != null ? Arrays.stream(rows).sum() : 0; - } - - @Override - public Integer batchInsertGeneAlias(List geneAliases) throws Exception { - String SQL = "INSERT INTO gene_alias " + - "(entrez_gene_id, gene_alias) " + - "VALUES(:entrezGeneId, :geneAlias)"; - int[] rows = namedParameterJdbcTemplate.batchUpdate(SQL, SqlParameterSourceUtils.createBatch(geneAliases.toArray(new GeneAlias[geneAliases.size()]))); - return rows != null ? Arrays.stream(rows).sum() : 0; - } - - /** - * Gene row mapper. - * @return - */ - private RowMapper geneRowMapper() { - return (RowMapper) (ResultSet rs, int i) -> { - Map beans_by_name = new HashMap(); - beans_by_name.put("gene", BeanMap.create(new Gene())); - - // go through each column from sql query and set respective bean properties accordingly - ResultSetMetaData resultSetMetaData = rs.getMetaData(); - for (int index=1; index<=resultSetMetaData.getColumnCount(); index++) { - // get the table name and the field name for the bean - String table = resultSetMetaData.getColumnLabel(index).split("\\.")[0]; - String field = resultSetMetaData.getColumnLabel(index).split("\\.")[1]; - - BeanMap beanMap = beans_by_name.get(table); - if (rs.getObject(index) != null) { - beanMap.put(field, rs.getObject(index)); - } - } - Gene gene = (Gene) beans_by_name.get("gene").getBean(); - gene.setAliases(getGeneAliases(gene.getEntrezGeneId())); - - return gene; - }; - } - -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/model/Gene.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/model/Gene.java deleted file mode 100644 index ed2eff4c0..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/model/Gene.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.model; - -import java.util.Set; - -/** - * - * @author ochoaa - */ -public class Gene { - - private Integer entrezGeneId; - private String hugoGeneSymbol; - private Integer geneticEntityId; - private String type; - private String cytoband; - private Integer length; - private Set aliases; - - public Gene() {} - - public Gene(Integer entrezGeneId, String hugoGeneSymbol, String type, String cytoband, Integer length, Set aliases) { - this.entrezGeneId = entrezGeneId; - this.hugoGeneSymbol = hugoGeneSymbol; - this.type = type; - this.cytoband = cytoband; - this.length = length; - this.aliases = aliases; - } - - /** - * @return the entrezGeneId - */ - public Integer getEntrezGeneId() { - return entrezGeneId; - } - - /** - * @param entrezGeneId the entrezGeneId to set - */ - public void setEntrezGeneId(Integer entrezGeneId) { - this.entrezGeneId = entrezGeneId; - } - - /** - * @return the hugoGeneSymbol - */ - public String getHugoGeneSymbol() { - return hugoGeneSymbol; - } - - /** - * @param hugoGeneSymbol the hugoGeneSymbol to set - */ - public void setHugoGeneSymbol(String hugoGeneSymbol) { - this.hugoGeneSymbol = hugoGeneSymbol; - } - - /** - * @return the geneticEntityId - */ - public Integer getGeneticEntityId() { - return geneticEntityId; - } - - /** - * @param geneticEntityId the geneticEntityId to set - */ - public void setGeneticEntityId(Integer geneticEntityId) { - this.geneticEntityId = geneticEntityId; - } - - /** - * @return the type - */ - public String getType() { - return type; - } - - /** - * @param type the type to set - */ - public void setType(String type) { - this.type = type; - } - - /** - * @return the cytoband - */ - public String getCytoband() { - return cytoband; - } - - /** - * @param cytoband the cytoband to set - */ - public void setCytoband(String cytoband) { - this.cytoband = cytoband; - } - - /** - * @return the length - */ - public Integer getLength() { - return length; - } - - /** - * @param length the length to set - */ - public void setLength(Integer length) { - this.length = length; - } - - /** - * @return the aliases - */ - public Set getAliases() { - return aliases; - } - - /** - * @param aliases the aliases to set - */ - public void setAliases(Set aliases) { - this.aliases = aliases; - } - -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/model/GeneAlias.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/model/GeneAlias.java deleted file mode 100644 index 7d2604f70..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/model/GeneAlias.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.model; - -import java.util.Objects; - -/** - * - * @author ochoaa - */ -public class GeneAlias { - - private Integer entrezGeneId; - private String geneAlias; - - public GeneAlias() {} - - public GeneAlias(Integer entrezGeneId, String geneAlias) { - this.entrezGeneId = entrezGeneId; - this.geneAlias = geneAlias; - } - - /** - * @return the entrezGeneId - */ - public Integer getEntrezGeneId() { - return entrezGeneId; - } - - /** - * @param entrezGeneId the entrezGeneId to set - */ - public void setEntrezGeneId(Integer entrezGeneId) { - this.entrezGeneId = entrezGeneId; - } - - /** - * @return the geneAlias - */ - public String getGeneAlias() { - return geneAlias; - } - - /** - * @param geneAlias the geneAlias to set - */ - public void setGeneAlias(String geneAlias) { - this.geneAlias = geneAlias; - } - - @Override - public int hashCode() { - int hash = 3; - hash = 53 * hash + (this.geneAlias != null ? this.geneAlias.hashCode() : 0); - hash = 53 * hash + (this.entrezGeneId != null ? this.entrezGeneId : 0); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (!GeneAlias.class.isAssignableFrom(obj.getClass())) { - return false; - } - final GeneAlias other = (GeneAlias) obj; - if (!Objects.equals(this.entrezGeneId, other.entrezGeneId)) { - return false; - } - if ((this.geneAlias == null ) ? (other.geneAlias != null) : !this.geneAlias.equals(other.geneAlias)) { - return false; - } - return true; - } -} diff --git a/gene/src/main/java/org/mskcc/cmo/ks/gene/util/GeneDataUtil.java b/gene/src/main/java/org/mskcc/cmo/ks/gene/util/GeneDataUtil.java deleted file mode 100644 index 45c14baa8..000000000 --- a/gene/src/main/java/org/mskcc/cmo/ks/gene/util/GeneDataUtil.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal CMO-Pipelines. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cmo.ks.gene.util; - -import org.mskcc.cmo.ks.gene.jdbc.internal.DaoGeneJdbcImpl; -import org.mskcc.cmo.ks.gene.model.Gene; - -import java.util.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Repository; - -/** - * - * @author ochoaa - */ -@Repository -public class GeneDataUtil { - - @Autowired - private DaoGeneJdbcImpl daoGeneJdbcImpl; - - @Bean(name="entrezGeneIdMap") - public Map entrezGeneIdMap() { - Map entrezGeneIdMap = new HashMap<>(); - for (Gene gene : daoGeneJdbcImpl.getAllGenes()) { - // fill gene symbol, gene id, and gene alias maps - entrezGeneIdMap.put(gene.getEntrezGeneId(), gene); - } - return entrezGeneIdMap; - } - - @Bean(name="ambiguousGenesMap") - public Map ambiguousGenesMap() { - Map ambiguousGenesMap = new HashMap<>(); - for (Gene gene : daoGeneJdbcImpl.getAllAmbiguousGenes()) { - ambiguousGenesMap.put(gene.getEntrezGeneId(), gene); - } - return ambiguousGenesMap; - } -} diff --git a/gene/src/main/resources/application.properties.EXAMPLE b/gene/src/main/resources/application.properties.EXAMPLE deleted file mode 100644 index a89335701..000000000 --- a/gene/src/main/resources/application.properties.EXAMPLE +++ /dev/null @@ -1,8 +0,0 @@ -spring.batch.job.enabled=false -chunk.interval=100 - -db.user= -db.password= -db.driver=com.mysql.jdbc.Driver -db.url= -db.use_ssl=false diff --git a/gene/src/main/resources/log4j.properties.EXAMPLE b/gene/src/main/resources/log4j.properties.EXAMPLE deleted file mode 100644 index 9d220dcfd..000000000 --- a/gene/src/main/resources/log4j.properties.EXAMPLE +++ /dev/null @@ -1,32 +0,0 @@ -# Change INFO to DEBUG, if you want to see debugging info on underlying libraries we use. -log4j.rootLogger=INFO, a - -# Change INFO to DEBUG, if you want see debugging info on our packages only. -log4j.category.org.mskcc=INFO -#log4j.category.org.springframework=ALL - -#log4j.logger.org.springframework.security=DEBUG -#log4j.logger.org.springframework.integration=DEBUG - - -# Use the JVM option, e.g.: "java -DPORTAL_HOME=/pathto/portal_homedir", -# or - "java -DPORTAL_HOME=$PORTAL_HOME", where PORTAL_HOME is shell (environment) variable. - -## IMPORTANT - THRESHOLD SHOULD NOT BE DEBUG FOR PRODUCTION, CREDENTIALS CAN BE DISPLAYED! - -log4j.appender.a = org.apache.log4j.FileAppender - -log4j.appender.a.File = /path/to/logfile.log - -log4j.appender.a.layout = org.apache.log4j.PatternLayout -log4j.appender.a.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} [%t] %-5p %c - %m%n -log4j.appender.a.append = true - -#log4j.logger.org.hibernate=INFO, a -#log4j.logger.org.hibernate.SQL=DEBUG -#log4j.logger.org.hibernate.type=TRACE -#log4j.logger.org.hibernate.hql.ast.AST=info -#log4j.logger.org.hibernate.tool.hbm2ddl=warn -#log4j.logger.org.hibernate.hql=debug -#log4j.logger.org.hibernate.cache=info -#log4j.logger.org.hibernate.jdbc=debug diff --git a/import-scripts/backup-eks-dbs.sh b/import-scripts/backup-eks-dbs.sh index cc3455851..456aa9621 100755 --- a/import-scripts/backup-eks-dbs.sh +++ b/import-scripts/backup-eks-dbs.sh @@ -21,7 +21,7 @@ for db in ${DBS[@]}; do $(set -o pipefail && mysqldump --login-path=mysql_localhost --quick $db | gzip > $SQLDUMP_FULLPATH) if [ $? -eq 0 ]; then echo "Successfully dumped: '${SQLDUMP_FILENAME}'" - . /data/portal-cron/git-repos/portal-configuration/eks-cluster/pipelines/authenticate_service_account.sh + . /data/portal-cron/git-repos/portal-configuration/eks-cluster/pipelines/authenticate_service_account.sh eks aws s3 cp ${SQLDUMP_FULLPATH} s3://cbioportal-backups/${SQLDUMP_FILENAME} --profile saml if [ $? -ne 0 ]; then echo "ERROR: failed to cp '${SQLDUMP_FILENAME}' to S3" diff --git a/import-scripts/clear_cbioportal_persistence_cache.sh b/import-scripts/clear_cbioportal_persistence_cache.sh index d10e394ac..ccaf8d8dd 100755 --- a/import-scripts/clear_cbioportal_persistence_cache.sh +++ b/import-scripts/clear_cbioportal_persistence_cache.sh @@ -77,9 +77,10 @@ fi unset KUBECONFIG_ARG if [ "$cluster_id" == "$CLUSTER_ID_DIGITS" ] ; then - /data/portal-cron/scripts/authenticate_service_account.sh + /data/portal-cron/scripts/authenticate_service_account.sh eks else if ! [ -z $PUBLIC_CLUSTER_KUBECONFIG ] ; then + /data/portal-cron/scripts/authenticate_service_account.sh public KUBECONFIG_ARG="--kubeconfig $PUBLIC_CLUSTER_KUBECONFIG" fi fi diff --git a/import-scripts/combine_files_py3.py b/import-scripts/combine_files_py3.py new file mode 100644 index 000000000..5966a8a70 --- /dev/null +++ b/import-scripts/combine_files_py3.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python + +""" combine_files_py3.py +This script merges an arbitrary number of files into one combined file. +Its primary use is to: +- merge DDP files from MSK-IMPACT, HEMEPACT, ACCESS to generate merged DDP files +for the msk_solid_heme cohort. This merged DDP file is only used for GENIE cohort creation. +- merge cvr/seq_date.txt files from MSK-IMPACT, HEMEPACT, ACCESS to generate a merged seq_date file +for the msk_solid_heme cohort. This merged seq_date.txt file is only used for the Sophia cohort. +Usage: + python combine_files_py3.py --input-files $FILE1 $FILE2 <...> --output-file $OUTPUT_FILE +Example: + python combine_files_py3.py --input-files mskimpact/ddp/ddp_naaccr.txt mskaccess/ddp/ddp_naaccr.txt mskimpact_heme/ddp/ddp_naaccr.txt \ + --output-file merged_ddp_naaccr.txt +""" + +import sys +import argparse +import os +import pandas as pd +from functools import reduce +import csv + +ERROR_FILE = sys.stderr + + +def write_tsv(df, path, **opts): + """ + Writes a TSV file to the given path. + """ + opts["index"] = opts.get( + "index", False + ) # by default, don't write the index column to the output + df.to_csv( + path, + sep="\t", + **opts, + ) + + +def combine_files(input_files, output_file, sep="\t", columns=None, merge_type="inner"): + data_frames = [] + for file in input_files: + df = pd.read_table( + file, + sep=sep, + comment="#", + float_precision="round_trip", + na_filter=False, + low_memory=False, + ) + data_frames.append(df) + + df_merged = reduce( + lambda left, right: pd.merge(left, right, on=columns, how=merge_type), data_frames + ) + write_tsv( + df_merged, + output_file, + quoting=csv.QUOTE_NONE, + ) + + +def main(): + parser = argparse.ArgumentParser(prog="combine_files_py3.py") + parser.add_argument( + "-i", + "--input-files", + dest="input_files", + action="store", + required=True, + nargs="+", + help="paths to files to combine", + ) + parser.add_argument( + "-o", + "--output-file", + dest="output_file", + action="store", + required=True, + help="output path for combined file", + ) + parser.add_argument( + "-s", + "--separator", + dest="sep", + action="store", + default="\t", + help="Character or regex pattern to treat as the delimiter", + ) + parser.add_argument( + "-c", + "--columns", + dest="columns", + action="store", + default=None, + nargs="+", + help="Column or index level names to join on. If None, defaults to the intersection of the columns in both DataFrames", + ) + parser.add_argument( + "-m", + "--merge-type", + dest="merge_type", + action="store", + default="inner", + help="Type of merge: {left, right, outer, inner, cross}, default: inner", + ) + + args = parser.parse_args() + input_files = args.input_files + output_file = args.output_file + sep = args.sep + columns = args.columns + merge_type = args.merge_type + + # Check that the input files exist + for file in input_files: + if not os.path.exists(file): + print(f"No such file: {file}", file=ERROR_FILE) + parser.print_help() + + # Combine the files + combine_files(input_files, output_file, sep=sep, columns=columns, merge_type=merge_type) + + +if __name__ == "__main__": + main() diff --git a/import-scripts/delete_slow_to_terminate_pods.sh b/import-scripts/delete_slow_to_terminate_pods.sh index 1c1eac70c..4d5534c64 100755 --- a/import-scripts/delete_slow_to_terminate_pods.sh +++ b/import-scripts/delete_slow_to_terminate_pods.sh @@ -183,7 +183,7 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/delete_slow_to_terminate_pods.loc mv $temp_file $POD_TERMINATING_TIMESTAMP_FILE } - /data/portal-cron/scripts/authenticate_service_account.sh + /data/portal-cron/scripts/authenticate_service_account.sh eks check_for_dependencies now=$(date +%s) make_work_dir_if_necessary diff --git a/import-scripts/detect_samples_with_problematic_metadata.py b/import-scripts/detect_samples_with_problematic_metadata.py index af6c3050f..a5b57e636 100755 --- a/import-scripts/detect_samples_with_problematic_metadata.py +++ b/import-scripts/detect_samples_with_problematic_metadata.py @@ -39,7 +39,7 @@ def sample_missing_required_metadata(metadata, problematic_metadata): if metadata[required_field] is None: problematic_metadata["gene-panel"] = None return True - if required_field == "gene-panel" and metadata[required_field].casefold() == "unknown": + if required_field == "gene-panel" and metadata[required_field].casefold() in {"unknown", "archer-solid-cv4"}: problematic_metadata["gene-panel"] = metadata[required_field] return True return False diff --git a/import-scripts/dmp-import-vars-functions.sh b/import-scripts/dmp-import-vars-functions.sh index 13602b84f..a1ccdc0c8 100755 --- a/import-scripts/dmp-import-vars-functions.sh +++ b/import-scripts/dmp-import-vars-functions.sh @@ -7,13 +7,11 @@ PERFORM_CRDB_FETCH=0 PROCESS_UNLINKED_ARCHER_STUDY=0 CRDB_FETCHER_JAR_FILENAME="$PORTAL_HOME/lib/crdb_fetcher.jar" CVR_FETCHER_JAR_FILENAME="$PORTAL_HOME/lib/cvr_fetcher.jar" -DARWIN_FETCHER_JAR_FILENAME="$PORTAL_HOME/lib/darwin_fetcher.jar" DDP_FETCHER_JAR_FILENAME="$PORTAL_HOME/lib/ddp_fetcher.jar" REDCAP_PIPELINE_JAR_FILENAME="$PORTAL_HOME/lib/redcap_pipeline.jar" IMPORTER_JAR_FILENAME="$PORTAL_HOME/lib/msk-dmp-importer.jar" JAVA_CRDB_FETCHER_ARGS="--add-opens java.base/java.lang=ALL-UNNAMED -jar $CRDB_FETCHER_JAR_FILENAME" JAVA_CVR_FETCHER_ARGS="-Xmx64g -jar $CVR_FETCHER_JAR_FILENAME" -JAVA_DARWIN_FETCHER_ARGS="--add-opens java.base/java.lang=ALL-UNNAMED -jar $DARWIN_FETCHER_JAR_FILENAME" JAVA_DDP_FETCHER_ARGS="-Xmx48g $JAVA_SSL_ARGS -jar $DDP_FETCHER_JAR_FILENAME" JAVA_REDCAP_PIPELINE_ARGS="$JAVA_SSL_ARGS -jar $REDCAP_PIPELINE_JAR_FILENAME" # the cvr server safety lockouts are no longer in use now that cvr timeout/retry loops are in effect @@ -166,13 +164,6 @@ function import_crdb_to_redcap { return $return_value } -# Function for importing mskimpact darwin files to redcap -function import_mskimpact_darwin_caisis_to_redcap { - return_value=0 - if ! import_project_to_redcap $MSK_IMPACT_DATA_HOME/data_clinical_supp_caisis_gbm.txt mskimpact_clinical_caisis ; then return_value=1 ; fi - return $return_value -} - # Function for importing mskimpact cvr files to redcap function import_mskimpact_cvr_to_redcap { return_value=0 diff --git a/import-scripts/fetch-ddp-and-import-cmo-access-data.sh b/import-scripts/fetch-ddp-and-import-cmo-access-data.sh index 81b83386e..a7e22ea28 100755 --- a/import-scripts/fetch-ddp-and-import-cmo-access-data.sh +++ b/import-scripts/fetch-ddp-and-import-cmo-access-data.sh @@ -227,7 +227,7 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-ddp-and-import-cmo-access-d sendClearCacheFailureMessage cmo-access fetch-ddp-and-import-cmo-access-data.sh fi else - echo "No studies have been updated, not clearing persitsence cache for cmo-access portal..." + echo "No studies have been updated, not clearing persistence cache for cmo-access portal..." fi # import ran and either failed or succeeded diff --git a/import-scripts/fetch-dmp-data-for-import.sh b/import-scripts/fetch-dmp-data-for-import.sh index c2ea4f93e..3adc17fa1 100755 --- a/import-scripts/fetch-dmp-data-for-import.sh +++ b/import-scripts/fetch-dmp-data-for-import.sh @@ -102,9 +102,6 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" fi fi - # pre-consume any samples missing values for normal_ad/tumor_dp/ad fields, or having UNKNOWN Archer gene-panel - bash $PORTAL_HOME/scripts/preconsume_problematic_samples.sh - # fetch clinical data from data repository echo "fetching updates from dmp repository..." $JAVA_BINARY $JAVA_IMPORTER_ARGS --fetch-data --data-source dmp --run-date latest @@ -188,6 +185,8 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" # fetch new/updated IMPACT samples using CVR Web service (must come after git fetching) drop_dead_instant_step=$(date --date="+3hours" -Iseconds) # nearly 3 hours from now drop_dead_instant_string=$(find_earlier_instant "$drop_dead_instant_step" "$DROP_DEAD_INSTANT_END_TO_END") + # pre-consume any samples missing values for normal_ad/tumor_dp/ad fields, or having UNKNOWN gene-panel + bash $PORTAL_HOME/scripts/preconsume_problematic_samples.sh mskimpact printTimeStampedDataProcessingStepMessage "CVR fetch for mskimpact" $JAVA_BINARY $JAVA_CVR_FETCHER_ARGS -d $MSK_IMPACT_DATA_HOME -p $MSK_IMPACT_PRIVATE_DATA_HOME -n data_clinical_mskimpact_data_clinical_cvr.txt -i mskimpact -r 150 $CVR_TEST_MODE_ARGS -z $drop_dead_instant_string if [ $? -gt 0 ] ; then @@ -265,6 +264,8 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" # fetch new/updated heme samples using CVR Web service (must come after git fetching). Threshold is set to 50 since heme contains only 190 samples (07/12/2017) drop_dead_instant_step=$(date --date="+3hours" -Iseconds) # nearly 3 hours from now drop_dead_instant_string=$(find_earlier_instant "$drop_dead_instant_step" "$DROP_DEAD_INSTANT_END_TO_END") + # pre-consume any samples missing values for normal_ad/tumor_dp/ad fields, or having UNKNOWN gene-panel + bash $PORTAL_HOME/scripts/preconsume_problematic_samples.sh mskimpact_heme printTimeStampedDataProcessingStepMessage "CVR fetch for hemepact" $JAVA_BINARY $JAVA_CVR_FETCHER_ARGS -d $MSK_HEMEPACT_DATA_HOME -p $MSK_HEMEPACT_PRIVATE_DATA_HOME -n data_clinical_hemepact_data_clinical.txt -i mskimpact_heme -r 50 $CVR_TEST_MODE_ARGS -z $drop_dead_instant_string if [ $? -gt 0 ] ; then @@ -329,6 +330,8 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" # fetch new/updated archer samples using CVR Web service (must come after git fetching). drop_dead_instant_step=$(date --date="+3hours" -Iseconds) # nearly 3 hours from now drop_dead_instant_string=$(find_earlier_instant "$drop_dead_instant_step" "$DROP_DEAD_INSTANT_END_TO_END") + # pre-consume any samples missing values for normal_ad/tumor_dp/ad fields, or having UNKNOWN gene-panel + bash $PORTAL_HOME/scripts/preconsume_problematic_samples.sh mskarcher printTimeStampedDataProcessingStepMessage "CVR fetch for archer" # archer has -b option to block warnings for samples with zero variants (all samples will have zero variants) $JAVA_BINARY $JAVA_CVR_FETCHER_ARGS -d $MSK_ARCHER_UNFILTERED_DATA_HOME -p $MSK_ARCHER_UNFILTERED_PRIVATE_DATA_HOME -n data_clinical_mskarcher_data_clinical.txt -i mskarcher -s -b -r 50 $CVR_TEST_MODE_ARGS -z $drop_dead_instant_string @@ -364,6 +367,8 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" # fetch new/updated access samples using CVR Web service (must come after git fetching). drop_dead_instant_step=$(date --date="+3hours" -Iseconds) # nearly 3 hours from now drop_dead_instant_string=$(find_earlier_instant "$drop_dead_instant_step" "$DROP_DEAD_INSTANT_END_TO_END") + # pre-consume any samples missing values for normal_ad/tumor_dp/ad fields, or having UNKNOWN gene-panel + bash $PORTAL_HOME/scripts/preconsume_problematic_samples.sh mskaccess printTimeStampedDataProcessingStepMessage "CVR fetch for access" # access has -b option to block warnings for samples with zero variants (all samples will have zero variants) $JAVA_BINARY $JAVA_CVR_FETCHER_ARGS -d $MSK_ACCESS_DATA_HOME -p $MSK_ACCESS_PRIVATE_DATA_HOME -n data_clinical_mskaccess_data_clinical.txt -i mskaccess -s -b -r 50 $CVR_TEST_MODE_ARGS -z $drop_dead_instant_string diff --git a/import-scripts/filter-clinical-arg-functions.sh b/import-scripts/filter-clinical-arg-functions.sh new file mode 100644 index 000000000..11bd91d68 --- /dev/null +++ b/import-scripts/filter-clinical-arg-functions.sh @@ -0,0 +1,107 @@ +unset clinical_attributes_in_file +declare -gA clinical_attributes_in_file=() +unset clinical_attributes_to_filter_arg +declare -g clinical_attributes_to_filter_arg="unset" + +function find_clinical_attribute_header_line_from_file() { + # Path to clinical file taken as an argument + clinical_attribute_filepath="$1" + + # Results are stored in this variable + declare -g clinical_attribute_header_line="unset" + + # Error if clinical file cannot be read + if ! [ -r "$clinical_attribute_filepath" ] ; then + echo "error: cannot read file $clinical_attribute_filepath" >&2 + return 1 + fi + + # Search file for header line + while read -r line ; do + if [ ${#line} -eq 0 ] ; then + echo "error: first uncommented line in $clinical_attribute_filepath was empty" >&2 + return 1 + fi + if ! [ ${line:0:1} == "#" ] ; then + clinical_attribute_header_line=$line + break + fi + done < "$clinical_attribute_filepath" + if [ "$clinical_attribute_header_line" == "unset" ] ; then + echo "error: unable to find header line in $clinical_attribute_filepath" >&2 + return 1 + fi +} + +function find_clinical_attributes_in_file() { + # Path to clinical file taken as an argument + clinical_attribute_filepath="$1" + + # Results (array of clinical attributes) are stored in this global array + clinical_attributes_in_file=() + if ! find_clinical_attribute_header_line_from_file "$clinical_attribute_filepath" ; then + return 1 + fi + for attribute in $clinical_attribute_header_line ; do + clinical_attributes_in_file[$attribute]+=1 + done +} + +function find_clinical_attributes_to_filter_arg() { + # Path to clinical file taken as an argument + clinical_attribute_filepath="$1" + + # Attributes to deliver in the clinical file + attributes_to_deliver="$2" + + declare -A clinical_attributes_to_filter=() + if ! find_clinical_attributes_in_file "$clinical_attribute_filepath" ; then + return 1 + fi + + # Populate delivered attributes for given file type + unset delivered_attributes + declare -A delivered_attributes=() + for attribute in $attributes_to_deliver ; do + delivered_attributes[$attribute]+=1 + done + + # Determine which clinical attributes we need to filter based on the attributes found in the file + for attribute in ${!clinical_attributes_in_file[@]} ; do + if [ -z ${delivered_attributes[$attribute]} ] ; then + clinical_attributes_to_filter[$attribute]+=1 + fi + done + + # Put the list attributes we want to filter in a comma separated string + clinical_attributes_to_filter_arg="" + list_size=0 + for attribute in ${!clinical_attributes_to_filter[@]} ; do + clinical_attributes_to_filter_arg="$clinical_attributes_to_filter_arg$attribute" + list_size=$(($list_size+1)) + if [ "$list_size" -lt ${#clinical_attributes_to_filter[@]} ] ; then + clinical_attributes_to_filter_arg="$clinical_attributes_to_filter_arg," + fi + done +} + +function filter_clinical_attribute_columns() { + # Path to clinical file taken as an argument + clinical_attribute_filepath="$1" + + # Attributes to deliver in the clinical file + attributes_to_deliver="$2" + + # Path to output clinical file taken as an argument + output_filepath="$3" + + # Determine which columns to exclude in the patient file + find_clinical_attributes_to_filter_arg "$clinical_attribute_filepath" "$attributes_to_deliver" + EXCLUDED_HEADER_FIELD_LIST="$clinical_attributes_to_filter_arg" + + # Filter out the columns we want to exclude + $PYTHON_BINARY $PORTAL_HOME/scripts/filter_clinical_data.py -c "$clinical_attribute_filepath" -e "$EXCLUDED_HEADER_FIELD_LIST" > "$output_filepath" && + + # Rewrite file with updated data + mv "$output_filepath" "$clinical_attribute_filepath" +} diff --git a/import-scripts/import-cmo-data-triage.sh b/import-scripts/import-cmo-data-triage.sh index a9518e78e..6e0c7600a 100755 --- a/import-scripts/import-cmo-data-triage.sh +++ b/import-scripts/import-cmo-data-triage.sh @@ -108,7 +108,7 @@ FLOCK_FILEPATH="/data/portal-cron/cron-lock/import-cmo-data-triage.lock" sendClearCacheFailureMessage triage import-cmo-data-triage.sh fi else - echo "No studies have been updated, not clearing persitsence cache for triage portal..." + echo "No studies have been updated, not clearing persistence cache for triage portal..." fi # import ran and either failed or succeeded diff --git a/import-scripts/import-devdb-data.sh b/import-scripts/import-devdb-data.sh index a58ec0e53..aea2560d8 100755 --- a/import-scripts/import-devdb-data.sh +++ b/import-scripts/import-devdb-data.sh @@ -111,7 +111,7 @@ FLOCK_FILEPATH="/data/portal-cron/cron-lock/import-devdb-data.lock" sendClearCacheFailureMessage devdb import-data-devdb.sh fi else - echo "No studies have been updated, not clearing persitsence cache for devdb portal..." + echo "No studies have been updated, not clearing persistence cache for devdb portal..." fi # import ran and either failed or succeeded diff --git a/import-scripts/import-hgnc-data.sh b/import-scripts/import-hgnc-data.sh index 8234bec12..15b2b4a2f 100755 --- a/import-scripts/import-hgnc-data.sh +++ b/import-scripts/import-hgnc-data.sh @@ -111,7 +111,7 @@ FLOCK_FILEPATH="/data/portal-cron/cron-lock/import-hgnc-data.lock" sendClearCacheFailureMessage hgnc import-data-hgnc.sh fi else - echo "No studies have been updated, not clearing persitsence cache for hgnc portal..." + echo "No studies have been updated, not clearing persistence cache for hgnc portal..." fi # import ran and either failed or succeeded diff --git a/import-scripts/merge_clinical_metadata_headers_py3.py b/import-scripts/merge_clinical_metadata_headers_py3.py new file mode 100755 index 000000000..9243544ca --- /dev/null +++ b/import-scripts/merge_clinical_metadata_headers_py3.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 + +"""cbioportal clinical data file metadata header merge/transfer program + +This program reads a set of clinical data files in order to construct +a dictionary of metadata headers for any present clinical attribute +columns. It then will read an input file which is missing the metadata +headers, will attach the appropriate metadata headers from the +constructed dictionary, and output the clinical data file with attached +metadata. + +Args are positional. +Arg 1 is a filepath to the main input file. This file should be a file + in the cbioportal file format for clinical attribute data + (such as data_clinical_sample.txt, or data_clinical_patient.txt), + with column name headers but missing metadata headers. +Arg 2 is a filepath to which the output should be written. The written + file will have the same contents as the main input file, with + attached metadata headers added. +Arg 3,4,... are a list of filepaths to any number of cbioportal clinical + files which do have attached metadata headers, collectively + covering all of the attribute headers present in the input file. + These files are parsed by this program to construct a dictionary + of clinical attribute metadata, which will be used to produce + the attached metadata in the output file. + +All files must be tab-delimited and rectangular (each line contains +the same number of columns as other lines). This program will properly +handle processing of files of a single type. There are three types of +clinical attribute files : patient-centric (data_clinical_patient.txt), +sample-centric (data_clinical_sample.txt) or combination (data_clinical +.txt). The first two types have 4 lines of attached metadata headers, +and the third has 5. Do not mix file types when running this program. + +The main use case for this program is for re-attaching the clinical +attribute metadata headers when using the merge.py script to merge +together clinical data from multiple studies. The output of merge.py +will be lacking clinical attribute metadata lines, but if all studies +which were merged contained clinical attribute metadata and used the +same types of files, this script will re-attach the metadata attributes +present in the source studies. + +If there is ambiguity (such as two studies having differing metadata +for the same clinical attribute), whichever study comes first in the +list of supplied metadata-containing filepaths (arg 3,4,...) will be +determative when constructing the metadata dictionary. + +------------------------------------------------------------------------ +""" + +import sys + +def print_usage(): + sys.stdout.write('usage: merge_clinical_metadata_headers_py3.py input_filepath output_filepath [metadata_header_containing_filepath ...]\n') + +def validate_args(argv): + # exit if argument count has no provided metadata_header_containing_filepath + if len(argv) < 4: + print_usage() + sys.exit(1) + +def verify_column_length(line, expected_len): + # exit if a line does not have the expected number of columns + if len(line.rstrip('\r\n').split('\t')) != expected_len: + raise Exception(f'expected {expected_len} columns but found otherwise in line "{line}"') + +def read_metadata_for_headers_in_file(header_to_metadata_map, filepath): + """adds metadata for each attribute in filepath into map + + filepath is opened and headers are read. For each column, if the + column header is not present yet in header_to_metadata_map, add + into the map a list of metadata values for the column header. + """ + + # get the metadata and column headers as lines + input_file = open(filepath, 'r') + metadata_header_lines = [] + header_column_line = None + while not header_column_line: + next_line = input_file.readline() + if not next_line: + intput_file.close() + sys.stderr.write(f'error : could not find header columns in metadata header containing file "{filepath}"\n') + sys.exit(1) + if next_line[0:1] != '#': + header_column_line = next_line + else: + metadata_header_lines.append(next_line[1:]) # strip off the comment character + input_file.close() + # break lines by tab into columns, and collect all metadata into a list of lists (by line) + header_column_list = header_column_line.rstrip('\r\n').split('\t') + metadata_value_list_list = [] + for metadata_header_line in metadata_header_lines: + verify_column_length(metadata_header_line, len(header_column_list)) + metadata_value_list = metadata_header_line.rstrip('\r\n').split('\t') + metadata_value_list_list.append(metadata_value_list) + # combine metadata by column and add to dictionary (header_to_metadata_map) + column_index = 0 + for header_column in header_column_list: + if header_column in header_to_metadata_map: + column_index = column_index + 1 + continue # if header column metadata is already defined (from previous file) skip + metadata_list_for_column = [] + for metadata_value_list in metadata_value_list_list: + metadata_list_for_column.append(metadata_value_list[column_index]) + header_to_metadata_map[header_column] = metadata_list_for_column + column_index = column_index + 1 + +def read_metadata_for_headers_in_files(metadata_header_containing_filepath_list): + """construct and return metadata dictionary + + iterate through all metadata_header_containing_filepath files, + adding metadata to dictionary (header_to_metadata_map) + """ + + header_to_metadata_map = {} + for filepath in metadata_header_containing_filepath_list: + read_metadata_for_headers_in_file(header_to_metadata_map, filepath) + return header_to_metadata_map + +def read_header_columns_from_file(filepath): + """find and return the list of headers from a file + + skip any metadata and return a list of attribute column names + """ + + input_file = open(filepath, 'r') + header_column_line = None + while not header_column_line: + next_line = input_file.readline() + if not next_line: + intput_file.close() + sys.stderr.write(f'error : could not find header columns in input file "{filepath}"\n') + sys.exit(1) + if next_line[0:1] != '#': + header_column_line = next_line + input_file.close() + header_column_list = header_column_line.rstrip('\r\n').split('\t') + return header_column_list + +def verify_all_metadata_headers_have_the_same_length(header_to_metadata_header_map): + """ check that the dictionary has uniform metadata linecount + + insure that user did not mix 4-line and 5-line + metadata_header_containing files when constructing the dictionary. + """ + + metadata_header_length = None + reference_column_header = None + for column_header in header_to_metadata_header_map: + if not metadata_header_length: + reference_column_header = column_header + metadata_header_length = len(header_to_metadata_header_map[column_header]) + else: + if metadata_header_length != len(header_to_metadata_header_map[column_header]): + sys.stderr.write(f'error : after encountering column {reference_column_header} with {metadata_header_length} metadata header values, column {column_header} was encountered with a different number\n') + sys.stderr.write(str(header_to_metadata_header_map[column_header])) + sys.stderr.write('\n') + sys.exit(1) + +def verify_all_columns_have_defined_metadata(header_column_list, header_to_metadata_header_map): + #check that each header column is in the constructed dictionary + for header_column in header_column_list: + if header_column not in header_to_metadata_header_map: + sys.stderr.write(f'error : header column "{header_column}" in input_filepath has no defined metadata values in any of the metadata_header_containing files provided\n') + +def write_metadata_headers(output_file, header_column_list, header_to_metadata_header_map): + """write the metadata lines for header columns to ouput_file + + construct each metadata header line by iterating through + header_column_list and looking up the appropriate metadata value + from the dictionary. Also insure that the dictionary has uniform + length (metadata header linecount), and use the dictionary to + determine how many header lines to output. Insure no column is + missing metadata. + """ + + verify_all_metadata_headers_have_the_same_length(header_to_metadata_header_map) + verify_all_columns_have_defined_metadata(header_column_list, header_to_metadata_header_map) + number_of_metadata_header_lines = len(header_to_metadata_header_map[header_column_list[0]]) + for line_number in range(0, number_of_metadata_header_lines): + metadata_value_list = [] + for header_column in header_column_list: + metadata_header_lines_for_column = header_to_metadata_header_map[header_column] + metadata_value_list.append(metadata_header_lines_for_column[line_number]) + output_file.write("#") + output_file.write("\t".join(metadata_value_list)) + output_file.write("\n") + +def write_header_columns_and_data(output_file, input_filepath): + """copy the contents of the input file to the output file + + no metadata is expected in the inputfile, so the input file + should contain only the header columns and the data lines. + """ + + input_file = open(input_filepath, "r") + output_file.write(input_file.read()) # transmit the entire input file into the output file + input_file.close() + +def write_output_file(output_filepath, input_filepath, header_column_list, header_to_metadata_header_map): + """output the constructed metadata lines, then the input file + + insure that at least one column exists (fail on empty files) + """ + + if len(header_column_list) == 0: + sys.stderr.write(f'error : no columns detected in input_filepath\n') + sys.exit(1) + output_file = open(output_filepath, "w") + write_metadata_headers(output_file, header_column_list, header_to_metadata_header_map) + write_header_columns_and_data(output_file, input_filepath) + output_file.close() + +def main(): + """handle arguments, construct dictionary, produce output + + construct the dictionary from the metadata_header_containing_files, + obtain the list of header columns from the input file, and write + output. + """ + + validate_args(sys.argv) + input_filepath = sys.argv[1] + output_filepath = sys.argv[2] + metadata_header_containing_filepath_list = sys.argv[3:] + header_to_metadata_header_map = read_metadata_for_headers_in_files(metadata_header_containing_filepath_list) + header_column_list = read_header_columns_from_file(input_filepath) + write_output_file(output_filepath, input_filepath, header_column_list, header_to_metadata_header_map) + +if __name__ == '__main__': + main() diff --git a/import-scripts/merge_ddp_files.py b/import-scripts/merge_ddp_files.py deleted file mode 100644 index 7ebf4eec5..000000000 --- a/import-scripts/merge_ddp_files.py +++ /dev/null @@ -1,119 +0,0 @@ -#! /usr/bin/env python - -""" merge_ddp_files.py -This script merges an arbitrary number of DDP files *of the same format* into one combined file. -Its primary use is to merge DDP files from MSK-IMPACT, HEMEPACT, ACCESS to generate merged DDP files -for the msk_solid_heme cohort. This merged DDP file is only used for GENIE cohort creation. -This script requires a clinical file (patient or sample) containing the masterlist of patients -included in the study. -Usage: - python merge_ddp_files.py --ddp-files $DDP_FILE1 $DDP_FILE2 <...> --clinical-file $CLINICAL_FILE --output-file $OUTPUT_FILE -Example: - python merge_ddp_files.py --ddp-files mskimpact/ddp/ddp_naaccr.txt mskaccess/ddp/ddp_naaccr.txt mskimpact_heme/ddp/ddp_naaccr.txt \ - --clinical-file msk_solid_heme/data_clinical_patient.txt --output-file merged_ddp_naaccr.txt -""" - -import sys -import argparse -import os - -import clinicalfile_utils - -ERROR_FILE = sys.stderr - - -def merge_ddp_files(ddp_files, clinical_file, output_file): - # Get set of patients from clinical file - patient_id_set = clinicalfile_utils.get_value_set_for_clinical_attribute(clinical_file, 'PATIENT_ID') - - header = [] - patient_id_index = None - to_write = [] - - # Process each DDP file - for file in ddp_files: - header_processed = False - - with open(file, 'r') as f: - for line in f.readlines(): - data = line.rstrip('\n').split('\t') - if line.startswith('#'): - # Skip metadata headers - continue - else: - if not header_processed: - # Store header information - if not header: - header = data - patient_id_index = clinicalfile_utils.get_index_for_column(header, 'PATIENT_ID') - if patient_id_index == -1: - raise IndexError('Unable to find PATIENT_ID column in DDP file %s' % (file)) - to_write.append(line.rstrip('\n')) - - # Check if header matches headers of other DDP files - if header and data != header: - raise ValueError('Cannot merge DDP file with differing header: %s.' % (file)) - - header_processed = True - continue - - # Only process the 'PATIENT_ID' column - # Check if patient id is in masterlist and isn't a duplicate - patient_id = data[patient_id_index] - line_to_write = '\t'.join(data) - if patient_id in patient_id_set and line_to_write not in to_write: - to_write.append(line_to_write) - - clinicalfile_utils.write_data_list_to_file(output_file, to_write) - - -def main(): - parser = argparse.ArgumentParser(prog='merge_ddp_files.py') - parser.add_argument( - '-d', - '--ddp-files', - dest='ddp_files', - action='store', - required=True, - nargs='+', - help='paths to DDP files to merge', - ) - parser.add_argument( - '-c', - '--clinical-file', - dest='clinical_file', - action='store', - required=True, - help='path to clinical file containing patient masterlist', - ) - parser.add_argument( - '-o', - '--output-file', - dest='output_file', - action='store', - required=True, - help='output path for merged DDP file', - ) - - args = parser.parse_args() - ddp_files = args.ddp_files - clinical_file = args.clinical_file - output_file = args.output_file - - # Check that the DDP files exist - for file in ddp_files: - if not os.path.exists(file): - print >> ERROR_FILE, 'No such DDP file: ' + file - parser.print_help() - - # Check that the clinical file exists - if not os.path.exists(clinical_file): - print >> ERROR_FILE, 'No such clinical file: ' + file - parser.print_help() - - # Merge the DDP files - merge_ddp_files(ddp_files, clinical_file, output_file) - - -if __name__ == '__main__': - main() diff --git a/import-scripts/merge_dremio_clinical_data_into_cmo_access.sh b/import-scripts/merge_dremio_clinical_data_into_cmo_access.sh new file mode 100755 index 000000000..38b023a91 --- /dev/null +++ b/import-scripts/merge_dremio_clinical_data_into_cmo_access.sh @@ -0,0 +1,401 @@ +#!/usr/bin/env bash + +# This script runs in either a dev mode or a prod mode. If running in dev mode, a separate dev mode +# clone of cmo-access is used as a source for cmo-access study data, otherwise the main (cmo-access) +# repo is used. Source files from cmo-access are copied into a temporary directory for mergining with +# data from dremio/SMILE. The dremio/SMILE data is then extracted using a go program which queries +# dremio for the relevant clinical/timeline data. merge.py is used to merge dremio/SMILE data into +# the cmo-access study and this merger is copied back into the source data repository. Then a +# symlink is created based on the study identifier and importing is done. The persistence cache is +# cleared. Finally cleanup and notification is handled. +# For prod mode runs, the updated data is commited back into the cmo-access repository, and there is +# a separate check then to make sure that while data was being fetched from dremio and merged, no +# new commits were pushed into the source repository (otherwise we cancel today's run) + +FLOCK_FILEPATH="/data/portal-cron/cron-lock/merge_dremio_clinical_data_into_cmo_access.lock" +{ + echo $(date) + + # check lock so that script executions do not overlap + if ! flock --nonblock --exclusive $flock_fd ; then + echo "Failure : could not acquire lock for $FLOCK_FILEPATH another instance of this process seems to still be running." + exit 1 + fi + + AUTOMATION_SCRIPT_FILEPATH=/data/portal-cron/scripts/automation-environment.sh + DMP_IMPORT_VARS_AND_FUNCTIONS_FILEPATH=/data/portal-cron/scripts/dmp-import-vars-functions.sh + CLEAR_PERSISTENCE_CACHE_SHELL_FUNCTIONS_FILEPATH=/data/portal-cron/scripts/clear-persistence-cache-shell-functions.sh + + # set up enivornment variables and temp directory + if ! [ -f $AUTOMATION_SCRIPT_FILEPATH ] ; then + message="automation-environment.sh could not be found, exiting..." >&2 + echo ${message} + exit 1 + fi + if ! [ -f $DMP_IMPORT_VARS_AND_FUNCTIONS_FILEPATH ] ; then + message="dmp-import-vars-functions.sh could not be found, exiting..." >&2 + echo ${message} + exit 1 + fi + if ! [ -f $CLEAR_PERSISTENCE_CACHE_SHELL_FUNCTIONS_FILEPATH ] ; then + message="clear-persistence-cache-shell-functions.sh could not be found, exiting..." >&2 + echo ${message} + exit 1 + fi + + source $AUTOMATION_SCRIPT_FILEPATH + source $DMP_IMPORT_VARS_AND_FUNCTIONS_FILEPATH + source $CLEAR_PERSISTENCE_CACHE_SHELL_FUNCTIONS_FILEPATH + + if [ -z "$PORTAL_HOME" ] ; then + message="could not run merge_dremio_clinical_data_into_cmo_access.sh : automation-environment.sh script must be run in order to set needed environment variables" + echo ${message} >&2 + exit 1 + fi + + DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR=/data/portal-cron/tmp/merge_dremio_clinical_to_cmo_access + DREMIO_CLINICAL_STAGING_DIRECTORY="${DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR}/dremio_cfdna_data" + CMO_ACCESS_STAGING_INPUT_DIRECTORY="${DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR}/cmo_access_data" + DREMIO_CLINICAL_OUTPUT_DIRECTORY="${DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR}/output_data" + DREMIO_CREDENTIALS_FILE="$PORTAL_HOME/pipelines-credentials/eks-account.credentials" + DREMIO_USERNAME="" + DREMIO_PASSWORD="" + IMPORTER_JAR_FILENAME="$PORTAL_HOME/lib/msk-cmo-importer.jar" + ENABLE_DEBUGGING=0 + JAVA_IMPORTER_ARGS="$JAVA_PROXY_ARGS $JAVA_SSL_ARGS -Dspring.profiles.active=dbcp -Djava.io.tmpdir=$CMO_ACCESS_TMPDIR -ea -cp $IMPORTER_JAR_FILENAME org.mskcc.cbio.importer.Admin" + ONCOTREE_VERSION_TO_USE="oncotree_candidate_release" + RUNMODE_PROD="runmode_prod" + RUNMODE_DEV="runmode_dev" + runmode=$RUNMODE_DEV + CMO_ACCESS_PROD_DATA_HOME="$PORTAL_DATA_HOME/cmo-access" + CMO_ACCESS_DEV_DATA_HOME="$PORTAL_DATA_HOME/cmo-access-dev-dup-clone" + clone_homedir="$CMO_ACCESS_DEV_DATA_HOME" + active_cmo_access_data_home="$CMO_ACCESS_DEV_DATA_HOME" + PROD_CANCER_STUDY_IDENTIFIER="mixed_msk_cfdna_research_access" + DEV_CANCER_STUDY_IDENTIFIER="mixed_msk_cfdna_research_access_dremio_dev" + active_cancer_study_identifier="$DEV_CANCER_STUDY_IDENTIFIER" + IMPORT_DEV_SYMLINK_FILEPATH="$CMO_ACCESS_PROD_DATA_HOME/mixed_msk_cfdna_research_access_dremio_dev" + IMPORT_PROD_SYMLINK_FILEPATH="$CMO_ACCESS_PROD_DATA_HOME/mixed_msk_cfdna_research_access" + PIPELINES_EMAIL_LIST="cbioportal-pipelines@cbioportal.org" + cmo_access_notification_file=$(mktemp $CMO_ACCESS_TMPDIR/cmo-access-portal-update-notification.$now.XXXXXX) + current_changeset_hash_at_head="" + + function set_runmode_active_vars() { + if [ "$runmode" == "$RUNMODE_PROD" ] ; then + active_cmo_access_data_home="$CMO_ACCESS_PROD_DATA_HOME" + active_cancer_study_identifier="$PROD_CANCER_STUDY_IDENTIFIER" + else + active_cmo_access_data_home="$CMO_ACCESS_DEV_DATA_HOME" + active_cancer_study_identifier="$DEV_CANCER_STUDY_IDENTIFIER" + fi + } + + function set_runmode_from_args() { + runmode=$RUNMODE_DEV + lastarg="" + for arg in $@ ; do + if [ "$arg" == "--runmode=prod" ] ; then + runmode=$RUNMODE_PROD + fi + if [ "$arg" == "prod" ] && [ "$lastarg" == "--runmode" ] ; then + runmode=$RUNMODE_PROD + fi + if [ "$arg" == "--runmode=dev" ] ; then + runmode=$RUNMODE_DEV + fi + if [ "$arg" == "dev" ] && [ "$lastarg" == "--runmode" ] ; then + runmode=$RUNMODE_DEV + fi + lastarg="$arg" + done + set_runmode_active_vars + } + + function safe_delete_directory_recursively() { + dir_path=$(readlink -f "$1") + if [[ -d "$dir_path" && "$dir_path" != "/" ]] ; then + rm -rf "$dir_path" + fi + } + + function prepare_tempdirs() { + if [ ! -d $DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR ] ; then + mkdir $DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR + if [ $? -ne 0 ] ; then + message="error : required temp directory does not exist and could not be created : $DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR" + echo ${message} >&2 + exit 1 + fi + else + # clear the old contents + for entry in "$DREMIO_CLINICAL_TO_CMO_ACCESS_TMPDIR"/* ; do + safe_delete_directory_recursively $entry + done + fi + mkdir "${DREMIO_CLINICAL_STAGING_DIRECTORY}" + mkdir "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" + if ! mkdir "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" ; then + echo "could not create tmp directories, for example ${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" >&2 + exit 1 + fi + } + + function set_dremio_credentials() { + if ! [ -r $DREMIO_CREDENTIALS_FILE ] ; then + echo "$DREMIO_CREDENTIALS_FILE could not be read, exiting..." + exit 1 + fi + DREMIO_USERNAME="$(grep -rh eks.account.name $DREMIO_CREDENTIALS_FILE | sed 's/eks.account.name=//g')" + DREMIO_PASSWORD="$(grep -rh eks.password $DREMIO_CREDENTIALS_FILE | sed 's/eks.password=//g')" + if [ -z $DREMIO_PASSWORD ] ; then + echo "DREMIO PASSWORD could not be obtained from $DREMIO_CREDENTIALS_FILE, exiting..." >&2 + exit 1 + fi + } + + function set_clone_homedir() { + clone_homedir="$CMO_ACCESS_DEV_DATA_HOME" + if [ "$runmode" == "$RUNMODE_PROD" ] ; then + clone_homedir="$CMO_ACCESS_PROD_DATA_HOME" + fi + } + + function update_git_repository_before_rsync() { + if ! $GIT_BINARY -C "${clone_homedir}" pull ; then + echo "unable to pull updates to data repository : ${clone_homedir}, exiting..." >&2 + exit 1 + fi + if ! $GIT_BINARY -C "${clone_homedir}" reset HEAD --hard ; then + echo "unable to reset to head of data repository : ${clone_homedir}, exiting..." >&2 + exit 1 + fi + + } + + function get_cmo_access_data() { + set_dremio_credentials + update_git_repository_before_rsync + if ! rsync -a "${clone_homedir}/mixed_MSK_cfDNA_RESEARCH_ACCESS/" "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" ; then + echo "unable to rsync data from data repository ${clone_homedir} to staging area ${CMO_ACCESS_STAGING_INPUT_DIRECTORY}, exiting..." >&2 + exit 1 + fi + } + + function purge_timeline_files_from_cmo_access() { + from_directory="$1" + darwin_based_timeline_files="data_timeline_ddp_chemotherapy.txt data_timeline_ddp_radiation.txt data_timeline_ddp_surgery.txt meta_timeline_ddp_chemotherapy.txt meta_timeline_ddp_radiation.txt meta_timeline_ddp_surgery.txt" + for filename in $darwin_based_timeline_files; do + rm "${from_directory}/$filename" + done + } + + function update_cmo_access_data() { + purge_timeline_files_from_cmo_access "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" + # construct the file "patient_id_mapping.txt" which maps cmo-patient-ids to dmp-patient-ids in a tab delimited table + old_dir=$(pwd) + cd "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" + if ! /data/portal-cron/bin/retrieve_cfdna_patient_id_mappings --host=tlvidreamcord1.mskcc.org --port=32010 --user=${DREMIO_USERNAME} --pass=${DREMIO_PASSWORD} ; then + echo "go program retrieve_cfdna_patient_id_mappings exited with non-zero status, exiting..." >&2 + exit 1 + fi + patient_mapping_filepath="${CMO_ACCESS_STAGING_INPUT_DIRECTORY}/patient_id_mapping.txt" + patient_mapping_ambiguous_filepath="${CMO_ACCESS_STAGING_INPUT_DIRECTORY}/patient_id_mapping_ambiguous.txt" + patient_mapping_filtered_filepath="${CMO_ACCESS_STAGING_INPUT_DIRECTORY}/patient_id_mapping_filtered.txt" + # apply the patient id mappings and convert matching cmo-patient-ids into dmp-patient-ids + if ! ${PORTAL_HOME}/scripts/update_cfdna_clinical_sample_patient_ids_via_dremio.sh data_clinical_sample.txt "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" "${patient_mapping_filepath}"; then + echo "script which applies patient_id_mapping file conversions to samples exited with non-zero status, exiting..." >&2 + exit 1 + fi + if ! ${PORTAL_HOME}/scripts/update_cfdna_clinical_sample_patient_ids_via_dremio.sh data_clinical_patient.txt "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" "${patient_mapping_filepath}"; then + echo "script which applies patient_id_mapping file conversions to patients exited with non-zero status, exiting..." >&2 + exit 1 + fi +#### rm -f "${patient_mapping_filepath}" "${patient_mapping_ambiguous_filepath}" "${patient_mapping_filtered_filepath}" + # replace the original clinical files with the updated/transformed versions which were output + mv data_clinical_sample.txt.updated data_clinical_sample.txt + mv data_clinical_patient.txt.updated data_clinical_patient.txt + cd "$old_dir" + } + + function rsync_files_to_active_repo() { + destination_dirpath="${clone_homedir}/mixed_MSK_cfDNA_RESEARCH_ACCESS" + if ! rsync -a "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}/" "${destination_dirpath}" ; then + echo "rsync of files from ${DREMIO_CLINICAL_OUTPUT_DIRECTORY} to ${destination_dirpath} failed, exiting..." >&2 + exit 1 + fi + } + + function set_current_changeset_hash_at_head() { + current_changeset_hash_at_head=$($GIT_BINARY -C "${clone_homedir}" log | head -n 1 | cut -f 2 -d $" ") + } + + function create_and_push_github_changeset() { + log_message_for_commit="$1" + set_current_changeset_hash_at_head + current_changeset_hash_at_head_previous="$current_changeset_hash_at_head" + if ! $GIT_BINARY -C "${clone_homedir}" pull ; then + echo "git pull on ${clone_homedir} failed, exiting..." >&2 + exit 1 + fi + if ! $GIT_BINARY -C "${clone_homedir}" reset HEAD --hard ; then + echo "git reset on ${clone_homedir} failed, exiting..." >&2 + exit 1 + fi + set_current_changeset_hash_at_head + if ! [ "$current_changeset_hash_at_head_previous" == "$current_changeset_hash_at_head" ] ; then + # there has been commits added since we began this run - report and exit + echo "The repository at ${clone_homedir} has been modified during the time that merging occurred. Exiting..." >&2 + exit 1 + fi + rsync_files_to_active_repo + if ! $GIT_BINARY -C "${clone_homedir}" add "${clone_homedir}/mixed_MSK_cfDNA_RESEARCH_ACCESS"/* ; then + echo "git add on ${clone_homdir} failed, exiting..." >&2 + exit 1 + fi + if ! $GIT_BINARY -C "${clone_homedir}" commit -m "${log_message_for_commit}" ; then + echo "git commit on ${clone_homdir} failed, exiting..." >&2 + exit 1 + fi + if ! $GIT_BINARY -C "${clone_homedir}" push origin ; then + sendPreImportFailureMessageMskPipelineLogsSlack "GIT PUSH (cmo_access) :fire: - address ASAP!" + echo "git push origin on ${clone_homdir} failed, exiting..." >&2 + exit 1 + fi + } + + function commit_updated_cmo_access_data() { + safe_delete_directory_recursively "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + mkdir "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + if [ "$runmode" == "$RUNMODE_PROD" ] ; then + # copy updated files to the output directory (so they are in position for the git commit function) + cp -a "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}/data_clinical_patient.txt" "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + cp -a "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}/data_clinical_sample.txt" "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + create_and_push_github_changeset "updated patient ids using dremio id preference logic" + fi + } + + function get_related_dremio_data() { + old_dir=$(pwd) + cd "${DREMIO_CLINICAL_STAGING_DIRECTORY}" + if ! /data/portal-cron/bin/convert_extract_cfdna --host=tlvidreamcord1.mskcc.org --port=32010 --user=${DREMIO_USERNAME} --pass=${DREMIO_PASSWORD} --sampleFile="${CMO_ACCESS_STAGING_INPUT_DIRECTORY}"/data_clinical_sample.txt ; then + echo "go program convert_extract_cfdna exited with non-zero status, exiting..." >&2 + exit 1 + fi + cd "$old_dir" + } + + function merge_clinical_data() { + safe_delete_directory_recursively "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + mkdir "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + if ! ${PYTHON_BINARY} ${PORTAL_HOME}/scripts/merge.py --output-directory "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" --study-id mixed_msk_cfdna_research_access --merge-clinical true --file-type-list clinical_patient,clinical_sample,timeline "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" "${DREMIO_CLINICAL_STAGING_DIRECTORY}" ; then + echo "merge.py exiting with non-zero status while merging dremio output into cmo-access study, exiting..." >&2 + exit 1 + fi + merged_clinical_sample_filepath="${DREMIO_CLINICAL_OUTPUT_DIRECTORY}/data_clinical_sample.txt" + merged_clinical_sample_with_metadata_filepath="${DREMIO_CLINICAL_OUTPUT_DIRECTORY}/data_clinical_sample.txt.with_metadata" + if ! ${PYTHON3_BINARY} ${PORTAL_HOME}/scripts/merge_clinical_metadata_headers_py3.py "$merged_clinical_sample_filepath" "$merged_clinical_sample_with_metadata_filepath" "$DREMIO_CLINICAL_STAGING_DIRECTORY/data_clinical_sample.txt" "$CMO_ACCESS_STAGING_INPUT_DIRECTORY/data_clinical_sample.txt" ; then + echo "merging of metadata heaers failed for ${merged_clinical_sample_filepath}, exiting..." >&2 + exit 1 + fi + merged_clinical_patient_filepath="${DREMIO_CLINICAL_OUTPUT_DIRECTORY}/data_clinical_patient.txt" + merged_clinical_patient_with_metadata_filepath="${DREMIO_CLINICAL_OUTPUT_DIRECTORY}/data_clinical_patient.txt.with_metadata" + if ! ${PYTHON3_BINARY} ${PORTAL_HOME}/scripts/merge_clinical_metadata_headers_py3.py "$merged_clinical_patient_filepath" "$merged_clinical_patient_with_metadata_filepath" "$DREMIO_CLINICAL_STAGING_DIRECTORY/data_clinical_patient.txt" "$CMO_ACCESS_STAGING_INPUT_DIRECTORY/data_clinical_patient.txt" ; then + echo "merging of metadata heaers failed for ${merged_clinical_patient_filepath}, exiting..." >&2 + exit 1 + fi + mv "$merged_clinical_sample_with_metadata_filepath" "$merged_clinical_sample_filepath" + mv "$merged_clinical_patient_with_metadata_filepath" "$merged_clinical_patient_filepath" + } + + function rsync_and_commit_merged_study_files() { + if ! [ "$runmode" == "$RUNMODE_PROD" ] ; then + # only rsync files when running in dev mode - skip commit + rsync_files_to_active_repo + return 0 + fi + create_and_push_github_changeset "Latest clinical data from Dremio/SMILE" + } + + function import_cmo_access_study() { + portal_name="cmo-access-dremio-dev-portal" + import_symlink_actual_target="${clone_homedir}/mixed_MSK_cfDNA_RESEARCH_ACCESS" + import_symlink_filepath="$IMPORT_DEV_SYMLINK_FILEPATH" + if [ "$runmode" == "$RUNMODE_PROD" ] ; then + portal_name="cmo-access-portal" + import_symlink_filepath="$IMPORT_PROD_SYMLINK_FILEPATH" + else + rm -f ${import_symlink_actual_target}/*ddp_chemotherapy* + rm -f ${import_symlink_actual_target}/*ddp_radiation* + rm -f ${import_symlink_actual_target}/*ddp_surgery* + fi + rm -f "$import_symlink_filepath" + ln -s "$import_symlink_actual_target" "$import_symlink_filepath" # symlink needed because the datasource search is based on study id, which is all lower case + # ----------------------------------------------------------------------------------------------------------- + # STUDY IMPORT + echo $(date) + # ------------------------------------------------------------- + printTimeStampedDataProcessingStepMessage "database version compatibility check" + $JAVA_BINARY $JAVA_IMPORTER_ARGS --check-db-version + if [ $? -gt 0 ] ; then + echo "Database version expected by portal does not match version in database!" + sendImportFailureMessageMskPipelineLogsSlack "MSK DMP Importer DB version check (CMO-ACCESS)" + EMAIL_BODY="The CMO-ACCESS database version is incompatible. Imports will be skipped until database is updated." + echo -e "Sending email $EMAIL_BODY" + echo -e "$EMAIL_BODY" | mail -s "MSKIMPACT Update Failure: DB version is incompatible" $PIPELINES_EMAIL_LIST + exit 1 + fi + echo "importing study data into msk-portal database..." + if ! $JAVA_BINARY -Xmx32G $JAVA_IMPORTER_ARGS --update-study-data --portal $portal_name --use-never-import --disable-redcap-export --notification-file "$cmo_access_notification_file" --oncotree-version ${ONCOTREE_VERSION_TO_USE} --transcript-overrides-source mskcc ; then + echo "CMO-ACCESS import failed!" + sendImportFailureMessageMskPipelineLogsSlack "CMO-ACCESS import failed!" + EMAIL_BODY="CMO-ACCESS import failed" + echo -e "Sending email $EMAIL_BODY" + echo -e "$EMAIL_BODY" | mail -s "Import failure: cmo_access" $PIPELINES_EMAIL_LIST + exit 1 + fi + rm -f "$import_symlink_filepath" + num_studies_updated=$(cat $CMO_ACCESS_TMPDIR/num_studies_updated.txt) + # clear persistence cache + if [[ $num_studies_updated -gt 0 ]]; then + echo "'$num_studies_updated' studies have been updated, clearing persistence cache for cmo-access portal..." + if ! clearPersistenceCachesForMskPortals ; then + sendClearCacheFailureMessage cmo-access merge_dremio_clinical_data_into_cmo_access.sh + fi + else + echo "No studies have been updated, not clearing persistence cache for cmo-access portal..." + fi + # import ran and either failed or succeeded + echo "sending notification email.." + #TODO we cannot rebuild importer currently, so use the mskimpact-portal which causes an email to be sent to our own group email only + $JAVA_BINARY $JAVA_IMPORTER_ARGS --send-update-notification --portal mskimpact-portal --notification-file "$cmo_access_notification_file" + } + + function remove_tempdirs() { + safe_delete_directory_recursively "${DREMIO_CLINICAL_STAGING_DIRECTORY}" + safe_delete_directory_recursively "${CMO_ACCESS_STAGING_INPUT_DIRECTORY}" + safe_delete_directory_recursively "${DREMIO_CLINICAL_OUTPUT_DIRECTORY}" + } + + function close_log_and_exit() { + echo "Fetching and importing of CMO-ACCESS complete!" + echo $(date) + } + + function main() { + set_runmode_from_args $@ + set_clone_homedir + prepare_tempdirs + get_cmo_access_data + update_cmo_access_data + commit_updated_cmo_access_data + get_related_dremio_data + merge_clinical_data + rsync_and_commit_merged_study_files + import_cmo_access_study + #remove_tempdirs + close_log_and_exit + } + + main $@ + +} {flock_fd}>$FLOCK_FILEPATH diff --git a/import-scripts/pipelines_eks/automation-environment.sh b/import-scripts/pipelines_eks/automation-environment.sh index 769248bf4..e9e16e0ed 100755 --- a/import-scripts/pipelines_eks/automation-environment.sh +++ b/import-scripts/pipelines_eks/automation-environment.sh @@ -132,6 +132,8 @@ export DATAHUB_DATA_HOME=$PORTAL_DATA_HOME/datahub/public export MSK_MIND_DATA_HOME=$PORTAL_DATA_HOME/msk-mind export MSK_SHAHLAB_DATA_HOME=$PORTAL_DATA_HOME/datahub_shahlab export CDSI_DATA_HOME=$PORTAL_DATA_HOME/cdm +export AZ_DATA_HOME=$PORTAL_DATA_HOME/az-data +export SOPHIA_DATA_HOME=$PORTAL_DATA_HOME/sophia-data ####################### # environment variables used across import scripts @@ -188,6 +190,7 @@ export LYMPHOMA_SUPER_COHORT_DATA_HOME=$DMP_DATA_HOME/lymphoma_super_cohort_fmi_ export MSK_EXTRACT_COHORT_DATA_HOME=$MSK_MIND_DATA_HOME/datahub/msk_extract_cohort2_2019 export MSK_SPECTRUM_COHORT_DATA_HOME=$MSK_SHAHLAB_DATA_HOME/msk_spectrum export MSK_CHORD_DATA_HOME=$CDSI_DATA_HOME/msk-chord +export AZ_MSK_IMPACT_DATA_HOME=$AZ_DATA_HOME/az_mskimpact # read-only data directories export FMI_BATLEVI_DATA_HOME=$FOUNDATION_DATA_HOME/mixed/lymphoma/mskcc/foundation/lymph_landscape_fmi_201611 # other data directories diff --git a/import-scripts/pipelines_eks/mycrontab b/import-scripts/pipelines_eks/mycrontab index b4e56ebb6..e9bd9f65c 100644 --- a/import-scripts/pipelines_eks/mycrontab +++ b/import-scripts/pipelines_eks/mycrontab @@ -16,6 +16,7 @@ MAILTO=cbioportal-pipelines@cbioportal.org 0 4,12 * * * . /data/portal-cron/scripts/automation-environment.sh;/data/portal-cron/scripts/import-public-data.sh >> /data/portal-cron/logs/import-public-data-msk.log 2>&1 || echo "Failure in crontab ($HOSTNAME): import-public-data.sh exited with non-zero exit status" | mail -r "cbioportal-pipelines@cbioportal.org" -s "Failure in crontab ($HOSTNAME)" cbioportal-pipelines@cbioportal.org 0 1 * * * . /data/portal-cron/scripts/automation-environment.sh ; touch ${START_TRIAGE_IMPORT_TRIGGER_FILENAME} 0 2 * * * . /data/portal-cron/scripts/automation-environment.sh;/data/portal-cron/scripts/fetch-ddp-and-import-cmo-access-data.sh >> /data/portal-cron/logs/fetch-ddp-and-import-cmo-access-data.log 2>&1 || echo "Failure in crontab ($HOSTNAME): fetch-ddp-and-import-cmo-access-data.sh exited with non-zero exit status" | mail -r "cbioportal-pipelines@cbioportal.org" -s "Failure in crontab ($HOSTNAME)" cbioportal-pipelines@cbioportal.org +0 4 * * * . /data/portal-cron/scripts/merge_dremio_clinical_data_into_cmo_access.sh --runmode=dev >> /data/portal-cron/logs/merge_dremio_clinical_data_into_cmo_access.log 2>&1 ########################## # Manual Import Script Launch Lines For Convenience (These should normally be commented out) ########################## diff --git a/import-scripts/pom.xml b/import-scripts/pom.xml index 5e2b84585..9052a0d47 100644 --- a/import-scripts/pom.xml +++ b/import-scripts/pom.xml @@ -4,12 +4,12 @@ MSKCC CMO Import Scripts Import Python Scripts import-scripts - 0.1.0 + 1.0.0 org.mskcc.cmo.ks master - 0.1.0 + 1.0.0 @@ -18,6 +18,7 @@ org.springframework.boot spring-boot-maven-plugin + ${spring.boot.version} true diff --git a/import-scripts/precache-portals.py b/import-scripts/precache-portals.py index b924c8326..b55b5e0db 100755 --- a/import-scripts/precache-portals.py +++ b/import-scripts/precache-portals.py @@ -30,7 +30,7 @@ def authenticate_service_account(): try: - subprocess.check_output(["/data/portal-cron/scripts/authenticate_service_account.sh"]) + subprocess.check_output(["/data/portal-cron/scripts/authenticate_service_account.sh public"]) except: print "Attempt to authenticate to k8s cluster failed with non-zero exit status, exiting..." sys.exit(1) diff --git a/import-scripts/preconsume_problematic_samples.sh b/import-scripts/preconsume_problematic_samples.sh index 0adf1a01c..fffb63fb7 100755 --- a/import-scripts/preconsume_problematic_samples.sh +++ b/import-scripts/preconsume_problematic_samples.sh @@ -1,24 +1,21 @@ #!/usr/bin/env bash -TMP_DIR="/data/portal-cron/tmp/preconsume_problematic_samples" +COHORT=$1 +FETCH_NUM=1 +TMP_DIR="/data/portal-cron/tmp/preconsume_problematic_samples/${COHORT}" CVR_FETCH_PROPERTIES_FILEPATH="/data/portal-cron/git-repos/pipelines-configuration/properties/fetch-cvr/application.properties" CVR_USERNAME=$(grep 'dmp.user_name' ${CVR_FETCH_PROPERTIES_FILEPATH} | head -n 1 | sed -E s/[^=][^=]*=//) CVR_PASSWORD=$(grep 'dmp.password' ${CVR_FETCH_PROPERTIES_FILEPATH} | head -n 1 | sed -E s/[^=][^=]*=//) CVR_TUMOR_SERVER=$(grep 'dmp.server_name' ${CVR_FETCH_PROPERTIES_FILEPATH} | head -n 1 | sed -E s/[^=][^=]*=//) CVR_CREATE_SESSION_URL="${CVR_TUMOR_SERVER}create_session/${CVR_USERNAME}/${CVR_PASSWORD}/0" +CVR_FETCH_URL_PREFIX="" CVR_IMPACT_FETCH_URL_PREFIX="${CVR_TUMOR_SERVER}cbio_retrieve_variants" CVR_HEME_FETCH_URL_PREFIX="${CVR_TUMOR_SERVER}cbio_retrieve_heme_variants" CVR_ARCHER_FETCH_URL_PREFIX="${CVR_TUMOR_SERVER}cbio_archer_retrieve_variants" CVR_ACCESS_FETCH_URL_PREFIX="${CVR_TUMOR_SERVER}cbio_retrieve_access_variants" CVR_CONSUME_SAMPLE_URL_PREFIX="${CVR_TUMOR_SERVER}cbio_consume_sample" -IMPACT_FETCH_OUTPUT_FILEPATH="$TMP_DIR/cvr_data_impact.json" -HEME_FETCH_OUTPUT_FILEPATH="$TMP_DIR/cvr_data_heme.json" -ARCHER_FETCH_OUTPUT_FILEPATH="$TMP_DIR/cvr_data_archer.json" -ACCESS_FETCH_OUTPUT_FILEPATH="$TMP_DIR/cvr_data_access.json" -IMPACT_CONSUME_IDS_FILEPATH="$TMP_DIR/impact_consume.ids" -HEME_CONSUME_IDS_FILEPATH="$TMP_DIR/heme_consume.ids" -ARCHER_CONSUME_IDS_FILEPATH="$TMP_DIR/archer_consume.ids" -ACCESS_CONSUME_IDS_FILEPATH="$TMP_DIR/access_consume.ids" +FETCH_OUTPUT_FILEPATH="" +CONSUME_IDS_FILEPATH="$TMP_DIR/consume.ids" PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH="$TMP_DIR/problematic_event_consume.ids" PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH="$TMP_DIR/problematic_metadata_consume.ids" CONSUME_ATTEMPT_OUTPUT_FILEPATH="$TMP_DIR/consume_attempt_output.json" @@ -33,36 +30,57 @@ function make_tmp_dir_if_necessary() { echo "Error : could not create tmp directory '$TMP_DIR'" >&2 exit 1 fi + else + # Remove files from last fetch + rm $TMP_DIR/* + fi +} + +function check_args() { + if [[ -z $COHORT ]] || [[ "$COHORT" != "mskimpact" && "$COHORT" != "mskimpact_heme" && "$COHORT" != "mskaccess" && "$COHORT" != "mskarcher" ]]; then + usage + exit 1 + fi +} + +function usage { + echo "preconsume_problematic_samples.sh \$COHORT_ID" + echo -e "\t\$COHORT_ID one of: ['mskimpact', 'mskimpact_heme', 'mskaccess', 'mskarcher']" +} + +function set_cvr_fetch_url_prefix() { + if [ "$COHORT" == "mskimpact" ] ; then + CVR_FETCH_URL_PREFIX=$CVR_IMPACT_FETCH_URL_PREFIX + elif [ "$COHORT" == "mskimpact_heme" ] ; then + CVR_FETCH_URL_PREFIX=$CVR_HEME_FETCH_URL_PREFIX + elif [ "$COHORT" == "mskarcher" ] ; then + CVR_FETCH_URL_PREFIX=$CVR_ARCHER_FETCH_URL_PREFIX + elif [ "$COHORT" == "mskaccess" ] ; then + CVR_FETCH_URL_PREFIX=$CVR_ACCESS_FETCH_URL_PREFIX fi } function fetch_currently_queued_samples() { + FETCH_OUTPUT_FILEPATH="$TMP_DIR/cvr_data_${FETCH_NUM}.json" dmp_token=$(curl $CVR_CREATE_SESSION_URL | grep session_id | sed -E 's/",[[:space:]]*$//' | sed -E 's/.*"//') - curl "${CVR_IMPACT_FETCH_URL_PREFIX}/${dmp_token}/0" > ${IMPACT_FETCH_OUTPUT_FILEPATH} - curl "${CVR_HEME_FETCH_URL_PREFIX}/${dmp_token}/0" > ${HEME_FETCH_OUTPUT_FILEPATH} - curl "${CVR_ARCHER_FETCH_URL_PREFIX}/${dmp_token}/0" > ${ARCHER_FETCH_OUTPUT_FILEPATH} - curl "${CVR_ACCESS_FETCH_URL_PREFIX}/${dmp_token}/0" > ${ACCESS_FETCH_OUTPUT_FILEPATH} + curl "${CVR_FETCH_URL_PREFIX}/${dmp_token}/0" > ${FETCH_OUTPUT_FILEPATH} } function detect_samples_with_problematic_events() { - $DETECT_SAMPLES_WITH_NULL_DP_AD_FIELDS_SCRIPT_FILEPATH ${IMPACT_FETCH_OUTPUT_FILEPATH} ${IMPACT_CONSUME_IDS_FILEPATH} - $DETECT_SAMPLES_WITH_NULL_DP_AD_FIELDS_SCRIPT_FILEPATH ${HEME_FETCH_OUTPUT_FILEPATH} ${HEME_CONSUME_IDS_FILEPATH} - $DETECT_SAMPLES_WITH_NULL_DP_AD_FIELDS_SCRIPT_FILEPATH ${ACCESS_FETCH_OUTPUT_FILEPATH} ${ACCESS_CONSUME_IDS_FILEPATH} + $DETECT_SAMPLES_WITH_NULL_DP_AD_FIELDS_SCRIPT_FILEPATH ${FETCH_OUTPUT_FILEPATH} ${PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH} } function detect_samples_with_problematic_metadata() { - $DETECT_SAMPLES_WITH_PROBLEMATIC_METADATA_SCRIPT_FILEPATH ${ARCHER_FETCH_OUTPUT_FILEPATH} ${ARCHER_CONSUME_IDS_FILEPATH} + $DETECT_SAMPLES_WITH_PROBLEMATIC_METADATA_SCRIPT_FILEPATH ${FETCH_OUTPUT_FILEPATH} ${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH} } -function combine_problematic_samples_for_consumption() { - cat ${IMPACT_CONSUME_IDS_FILEPATH} ${HEME_CONSUME_IDS_FILEPATH} ${ACCESS_CONSUME_IDS_FILEPATH} > ${PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH} - cat ${ARCHER_CONSUME_IDS_FILEPATH} > ${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH} -} - -function exit_if_no_problems_detected() { +function problems_were_detected() { if [ ! -s ${PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH} ] && [ ! -s ${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH} ] ; then - echo "no problematic samples detected .. exiting" - exit 0 + echo "no problematic samples detected" + return 1 + else + echo "problematic samples were detected" + return 0 fi } @@ -98,25 +116,31 @@ function attempt_to_consume_problematic_sample() { dmp_token="$1" sample_id="$2" type_of_problem="$3" # pass 'e' for event problems and 'm' for metadata problems + register_attempt="$4" HTTP_STATUS=$(curl -sSL -w '%{http_code}' -o "$CONSUME_ATTEMPT_OUTPUT_FILEPATH" "${CVR_CONSUME_SAMPLE_URL_PREFIX}/${dmp_token}/${sample_id}") if [[ $HTTP_STATUS =~ ^2 ]] ; then if ! grep '"error": "' "$CONSUME_ATTEMPT_OUTPUT_FILEPATH" ; then if grep --silent 'affectedRows": 1' "$CONSUME_ATTEMPT_OUTPUT_FILEPATH" ; then - register_successful_consumption "${sample_id}" "$type_of_problem" - continue + if [ "$register_attempt" == true ] ; then + register_successful_consumption "${sample_id}" "$type_of_problem" + continue + fi fi fi fi - register_failed_consumption "${sample_id}" "$type_of_problem" + if [ "$register_attempt" == true ] ; then + register_failed_consumption "${sample_id}" "$type_of_problem" + fi } function attempt_to_consume_problematic_samples() { + register_attempt=${1:-true} dmp_token=$(curl $CVR_CREATE_SESSION_URL | grep session_id | sed -E 's/",[[:space:]]*$//' | sed -E 's/.*"//') while read sample_id ; do - attempt_to_consume_problematic_sample "$dmp_token" "$sample_id" "e" + attempt_to_consume_problematic_sample "$dmp_token" "$sample_id" "e" "$register_attempt" done < ${PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH} while read sample_id ; do - attempt_to_consume_problematic_sample "$dmp_token" "$sample_id" "m" + attempt_to_consume_problematic_sample "$dmp_token" "$sample_id" "m" "$register_attempt" done < ${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH} } @@ -124,14 +148,28 @@ function consume_hardcoded_samples() { rm -f ${PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH} ${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH} touch ${PROBLEMATIC_EVENT_CONSUME_IDS_FILEPATH} touch ${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH} - echo "P-0025907-N01-IM6" >> "${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH}" + if [ "$COHORT" == "mskimpact" ] ; then + echo "P-0025907-N01-IM6" >> "${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH}" + fi if [ -f "${PROBLEMATIC_METADATA_CONSUME_IDS_FILEPATH}" ] ; then - attempt_to_consume_problematic_samples + # Won't register attempt (so it doesn't show up in logs every night) + attempt_to_consume_problematic_samples false fi } +function need_to_log_actions { + if [ ${#succeeded_to_consume_problematic_events_sample_list[@]} -gt 0 ] || \ + [ ${#failed_to_consume_problematic_events_sample_list[@]} -gt 0 ] || \ + [ ${#succeeded_to_consume_problematic_metadata_sample_list[@]} -gt 0 ] || \ + [ ${#failed_to_consume_problematic_metadata_sample_list[@]} -gt 0 ] ; then + return 0 + fi + return 1 +} + function log_actions() { date + echo -e "${COHORT^^} Problematic Samples" echo -e "consumed the following samples with problematic events:\n${succeeded_to_consume_problematic_events_sample_list[*]}" echo -e "attempted but failed to consume the following samples with problematic events:\n${failed_to_consume_problematic_events_sample_list[*]}" echo -e "consumed the following samples with problematic metadata:\n${succeeded_to_consume_problematic_metadata_sample_list[*]}" @@ -140,11 +178,11 @@ function log_actions() { } function post_slack_message() { - MESSAGE="<@U02D0Q0RWUE> <@U03FERRJ6SE> Warning : the following samples have been preemptively consumed before fetch because they contained events which required a value for fields {normal_dp, normal_ad, tumor_dp, tumor_ad} but contained no value in one or more of these fields.\nSuccessfully Consumed :\n${succeeded_to_consume_problematic_events_sample_list[*]}" + MESSAGE="<@U02D0Q0RWUE> <@U03FERRJ6SE> ${COHORT^^} Problematic Samples \nWarning : \nThe following samples have been preemptively consumed before fetch because they contained events which required a value for fields {normal_dp, normal_ad, tumor_dp, tumor_ad} but contained no value in one or more of these fields.\nSuccessfully Consumed :\n${succeeded_to_consume_problematic_events_sample_list[*]}" if [ ${#failed_to_consume_problematic_events_sample_list[@]} -gt 0 ]; then MESSAGE="${MESSAGE} Attempted Unsuccessfully To Consume :\n${failed_to_consume_problematic_events_sample_list[*]}" fi - MESSAGE="${MESSAGE}Warning : the following samples have been reemptively consumed before fetch because they contained problematic metadata where the gene-panel property was unset or had value UNKNOWN.\nSuccessfully Consumed :\n${succeeded_to_consume_problematic_metadata_sample_list[*]}" + MESSAGE="${MESSAGE}Warning : the following samples have been preemptively consumed before fetch because they contained problematic metadata where the gene-panel property was unset, invalid, or had value UNKNOWN.\nSuccessfully Consumed :\n${succeeded_to_consume_problematic_metadata_sample_list[*]}" if [ ${#failed_to_consume_problematic_metadata_sample_list[@]} -gt 0 ]; then MESSAGE="${MESSAGE} Attempted Unsuccessfully To Consume :\n${failed_to_consume_problematic_metadata_sample_list[*]}" fi @@ -152,21 +190,26 @@ function post_slack_message() { } date +check_args make_tmp_dir_if_necessary -failed_to_consume_problematic_events_sample_list=() # temporary code -succeeded_to_consume_problematic_events_sample_list=() # temporary code -failed_to_consume_problematic_metadata_sample_list=() # temporary code -succeeded_to_consume_problematic_metadata_sample_list=() # temporary code -consume_hardcoded_samples # temporary code -fetch_currently_queued_samples -detect_samples_with_problematic_events -detect_samples_with_problematic_metadata -combine_problematic_samples_for_consumption -exit_if_no_problems_detected +set_cvr_fetch_url_prefix failed_to_consume_problematic_events_sample_list=() succeeded_to_consume_problematic_events_sample_list=() failed_to_consume_problematic_metadata_sample_list=() succeeded_to_consume_problematic_metadata_sample_list=() -attempt_to_consume_problematic_samples -log_actions -post_slack_message +while : +do + consume_hardcoded_samples # temporary code + fetch_currently_queued_samples + detect_samples_with_problematic_events + detect_samples_with_problematic_metadata + if ! problems_were_detected ; then + break + fi + attempt_to_consume_problematic_samples + ((FETCH_NUM++)) +done +if need_to_log_actions ; then + log_actions + post_slack_message +fi \ No newline at end of file diff --git a/import-scripts/remove-duplicate-maf-variants.py b/import-scripts/remove-duplicate-maf-variants.py new file mode 100644 index 000000000..5a6abb6bb --- /dev/null +++ b/import-scripts/remove-duplicate-maf-variants.py @@ -0,0 +1,89 @@ +#!/usr/bin/python + +""" remove-duplicate-maf-variants.py +Script to remove duplicate maf records based on the 8 key columns. +Calculates VAF for each record and picks the record with high VAF +Formula for VAF = t_alt_count / (t_ref_count + t_alt_count) +""" + +import sys +import optparse + +ERROR_FILE = sys.stderr +OUTPUT_FILE = sys.stdout + +KEY_COLUMNS_INDEX = [] +KEY_COLUMNS = ['Entrez_Gene_Id','Chromosome','Start_Position','End_Position','Variant_Classification','Tumor_Seq_Allele2','Tumor_Sample_Barcode','HGVSp_Short'] +MAF_DATA = {} + +def remove_duplicate_variants(out_filename, comments, header, t_refc_index, t_altc_index): + outfile = [] + outfile.append(comments) + outfile.append(header) + for key in MAF_DATA: + if len(MAF_DATA[key]) > 1: + vaf_ind = 0 + vaf_value = 0 + for val in MAF_DATA[key]: + #calculate VAF for each duplicate record. + columns = val.rstrip('\n').split('\t') + try: + VAF = int(columns[t_altc_index])/(int(columns[t_altc_index])+int(columns[t_refc_index])) + if VAF > vaf_value: + vaf_value = VAF + vaf_ind = MAF_DATA[key].index(val) + outfile.append(MAF_DATA[key][vaf_ind]) + except: + print >> ERROR_FILE, 'ERROR: VAF cannot be calculated for the variant : ' + key + print >> ERROR_FILE, 'The t_ref_count is: '+ columns[t_refc_index]+ ' and t_alt_count is: '+ columns[t_altc_index] + outfile.append(val) + else: + outfile.append(MAF_DATA[key][0]) + + datafile = open(out_filename, 'w') + for line in outfile: + datafile.write(line) + datafile.close() + print >> OUTPUT_FILE, 'MAF file with duplicate variants removed is written to: ' + out_filename +'\n' + + +def main(): + # get command line arguments + parser = optparse.OptionParser() + parser.add_option('-i', '--input-maf-file', action = 'store', dest = 'input_maf_file') + parser.add_option('-o', '--output-maf-file', action = 'store', dest = 'output_maf_file') + + (options, args) = parser.parse_args() + maf_filename = options.input_maf_file + out_filename = options.output_maf_file + + comments = "" + header = "" + + with open(maf_filename,'r') as maf_file: + for line in maf_file: + if line.startswith('#'): + comments += line + elif line.startswith('Hugo_Symbol'): + header += line + header_cols = line.rstrip('\n').split('\t') + #get the positions of the 8 key maf columns + for value in KEY_COLUMNS: + KEY_COLUMNS_INDEX.append(header_cols.index(value)) + t_refc_index = header_cols.index('t_ref_count') + t_altc_index = header_cols.index('t_alt_count') + else: + reference_key = "" + data = line.rstrip('\n').split('\t') + for index in KEY_COLUMNS_INDEX: + reference_key += data[index]+'\t' + reference_key = reference_key.rstrip('\t') + if reference_key not in MAF_DATA: + MAF_DATA[reference_key] = [line] + else: + MAF_DATA[reference_key].append(line) + + remove_duplicate_variants(out_filename, comments, header, t_refc_index, t_altc_index) + +if __name__ == '__main__': + main() diff --git a/import-scripts/test/resources/merge_ddp_files/diff_headers/data_clinical_patient.txt b/import-scripts/test-py3/resources/combine_files/diff_headers/data_clinical_patient.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/diff_headers/data_clinical_patient.txt rename to import-scripts/test-py3/resources/combine_files/diff_headers/data_clinical_patient.txt diff --git a/import-scripts/test/resources/merge_ddp_files/diff_headers/ddp_naaccr.txt b/import-scripts/test-py3/resources/combine_files/diff_headers/ddp_naaccr.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/diff_headers/ddp_naaccr.txt rename to import-scripts/test-py3/resources/combine_files/diff_headers/ddp_naaccr.txt diff --git a/import-scripts/test/resources/merge_ddp_files/diff_headers/ddp_vital_status.txt b/import-scripts/test-py3/resources/combine_files/diff_headers/ddp_vital_status.txt similarity index 67% rename from import-scripts/test/resources/merge_ddp_files/diff_headers/ddp_vital_status.txt rename to import-scripts/test-py3/resources/combine_files/diff_headers/ddp_vital_status.txt index 3dfb55c71..27fcfd92b 100644 --- a/import-scripts/test/resources/merge_ddp_files/diff_headers/ddp_vital_status.txt +++ b/import-scripts/test-py3/resources/combine_files/diff_headers/ddp_vital_status.txt @@ -1,3 +1,4 @@ PATIENT_ID YEAR_CONTACT YEAR_DEATH INT_CONTACT INT_DOD DEAD P-000000F A B C D E -P-000000G A B C D E \ No newline at end of file +P-000000G A B C D E +P-000000E A B C D E \ No newline at end of file diff --git a/import-scripts/test-py3/resources/combine_files/diff_headers/expected.txt b/import-scripts/test-py3/resources/combine_files/diff_headers/expected.txt new file mode 100644 index 000000000..061e71b19 --- /dev/null +++ b/import-scripts/test-py3/resources/combine_files/diff_headers/expected.txt @@ -0,0 +1,8 @@ +PATIENT_ID NAACCR_SEX_CODE NAACCR_RACE_CODE_PRIMARY NAACCR_RACE_CODE_SECONDARY NAACCR_RACE_CODE_TERTIARY NAACCR_ETHNICITY_CODE BIRTH_YEAR YEAR_CONTACT YEAR_DEATH INT_CONTACT INT_DOD DEAD +P-000000A A B C D E F +P-000000B A B C D E F +P-000000C A B C D E F +P-000000D A B C D E F +P-000000E A B C D E F A B C D E +P-000000F A B C D E +P-000000G A B C D E diff --git a/import-scripts/test/resources/merge_ddp_files/duplicates/data_clinical_patient.txt b/import-scripts/test-py3/resources/combine_files/duplicates/data_clinical_patient.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/duplicates/data_clinical_patient.txt rename to import-scripts/test-py3/resources/combine_files/duplicates/data_clinical_patient.txt diff --git a/import-scripts/test/resources/merge_ddp_files/duplicates/ddp_naaccr1.txt b/import-scripts/test-py3/resources/combine_files/duplicates/ddp_naaccr1.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/duplicates/ddp_naaccr1.txt rename to import-scripts/test-py3/resources/combine_files/duplicates/ddp_naaccr1.txt diff --git a/import-scripts/test/resources/merge_ddp_files/duplicates/ddp_naaccr2.txt b/import-scripts/test-py3/resources/combine_files/duplicates/ddp_naaccr2.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/duplicates/ddp_naaccr2.txt rename to import-scripts/test-py3/resources/combine_files/duplicates/ddp_naaccr2.txt diff --git a/import-scripts/test/resources/merge_ddp_files/duplicates/ddp_naaccr3.txt b/import-scripts/test-py3/resources/combine_files/duplicates/ddp_naaccr3.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/duplicates/ddp_naaccr3.txt rename to import-scripts/test-py3/resources/combine_files/duplicates/ddp_naaccr3.txt diff --git a/import-scripts/test/resources/merge_ddp_files/duplicates/expected.txt b/import-scripts/test-py3/resources/combine_files/duplicates/expected.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/duplicates/expected.txt rename to import-scripts/test-py3/resources/combine_files/duplicates/expected.txt diff --git a/import-scripts/test/resources/merge_ddp_files/has_metadata_headers/data_clinical_patient.txt b/import-scripts/test-py3/resources/combine_files/has_metadata_headers/data_clinical_patient.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/has_metadata_headers/data_clinical_patient.txt rename to import-scripts/test-py3/resources/combine_files/has_metadata_headers/data_clinical_patient.txt diff --git a/import-scripts/test/resources/merge_ddp_files/has_metadata_headers/ddp_vital_status1.txt b/import-scripts/test-py3/resources/combine_files/has_metadata_headers/ddp_vital_status1.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/has_metadata_headers/ddp_vital_status1.txt rename to import-scripts/test-py3/resources/combine_files/has_metadata_headers/ddp_vital_status1.txt diff --git a/import-scripts/test/resources/merge_ddp_files/has_metadata_headers/ddp_vital_status2.txt b/import-scripts/test-py3/resources/combine_files/has_metadata_headers/ddp_vital_status2.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/has_metadata_headers/ddp_vital_status2.txt rename to import-scripts/test-py3/resources/combine_files/has_metadata_headers/ddp_vital_status2.txt diff --git a/import-scripts/test/resources/merge_ddp_files/has_metadata_headers/expected.txt b/import-scripts/test-py3/resources/combine_files/has_metadata_headers/expected.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/has_metadata_headers/expected.txt rename to import-scripts/test-py3/resources/combine_files/has_metadata_headers/expected.txt diff --git a/import-scripts/test/resources/merge_ddp_files/not_on_masterlist/data_clinical_patient.txt b/import-scripts/test-py3/resources/combine_files/not_on_masterlist/data_clinical_patient.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/not_on_masterlist/data_clinical_patient.txt rename to import-scripts/test-py3/resources/combine_files/not_on_masterlist/data_clinical_patient.txt diff --git a/import-scripts/test/resources/merge_ddp_files/not_on_masterlist/ddp_naaccr1.txt b/import-scripts/test-py3/resources/combine_files/not_on_masterlist/ddp_naaccr1.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/not_on_masterlist/ddp_naaccr1.txt rename to import-scripts/test-py3/resources/combine_files/not_on_masterlist/ddp_naaccr1.txt diff --git a/import-scripts/test/resources/merge_ddp_files/not_on_masterlist/ddp_naaccr2.txt b/import-scripts/test-py3/resources/combine_files/not_on_masterlist/ddp_naaccr2.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/not_on_masterlist/ddp_naaccr2.txt rename to import-scripts/test-py3/resources/combine_files/not_on_masterlist/ddp_naaccr2.txt diff --git a/import-scripts/test/resources/merge_ddp_files/not_on_masterlist/expected.txt b/import-scripts/test-py3/resources/combine_files/not_on_masterlist/expected.txt similarity index 100% rename from import-scripts/test/resources/merge_ddp_files/not_on_masterlist/expected.txt rename to import-scripts/test-py3/resources/combine_files/not_on_masterlist/expected.txt diff --git a/import-scripts/test-py3/test_combine_files.py b/import-scripts/test-py3/test_combine_files.py new file mode 100644 index 000000000..d955703f0 --- /dev/null +++ b/import-scripts/test-py3/test_combine_files.py @@ -0,0 +1,81 @@ +"""Provides unit tests for the test_merge_ddp_files.py script functionality. + +To run the tests, execute the following command from the parent +directory of this script: + python -m unittest discover test-py3 +""" + +import unittest +import os + +from combine_files_py3 import combine_files + +class TestCombineFiles(unittest.TestCase): + + # Show the diff even if it is very big + maxDiff = None + + @classmethod + def setUpClass(cls): + cls.base_dir = 'test-py3/resources/combine_files' + + def test_duplicates(self): + sub_dir = 'duplicates' + ddp_files = [ + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_naaccr1.txt'), + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_naaccr2.txt'), + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_naaccr3.txt'), + ] + merge_type='outer' + self.compare_expected_output_to_actual(sub_dir, ddp_files, merge_type) + + """ + def test_not_on_masterlist(self): + sub_dir = 'not_on_masterlist' + ddp_files = [ + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_naaccr1.txt'), + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_naaccr2.txt'), + ] + self.compare_expected_output_to_actual(sub_dir, ddp_files) + """ + + def test_diff_headers(self): + sub_dir = 'diff_headers' + ddp_files = [ + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_naaccr.txt'), + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_vital_status.txt'), + ] + merge_type='outer' + self.compare_expected_output_to_actual(sub_dir, ddp_files, merge_type) + + def test_has_metadata_headers(self): + sub_dir = 'has_metadata_headers' + ddp_files = [ + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_vital_status1.txt'), + os.path.join(TestCombineFiles.base_dir, sub_dir, 'ddp_vital_status2.txt'), + ] + merge_type='outer' + self.compare_expected_output_to_actual(sub_dir, ddp_files, merge_type) + + def compare_expected_output_to_actual(self, sub_dir, ddp_files, merge_type='inner', columns=None): + output_file = os.path.join(TestCombineFiles.base_dir, sub_dir, 'merged.txt') + expected_file = os.path.join(TestCombineFiles.base_dir, sub_dir, 'expected.txt') + + try: + combine_files(ddp_files, output_file, merge_type=merge_type, columns=None) + except (KeyError, ValueError): + if os.path.exists(output_file): + os.remove(output_file) + raise + + # Read output file and compare it to expected output + with open(expected_file, 'r') as expected_out: + with open(output_file, 'r') as actual_out: + self.assertEqual(expected_out.read(), actual_out.read()) + + # Clean up sample output file + os.remove(output_file) + + +if __name__ == '__main__': + unittest.main() diff --git a/import-scripts/test/resources/merge_ddp_files/no_patientid/data_clinical_patient.txt b/import-scripts/test/resources/merge_ddp_files/no_patientid/data_clinical_patient.txt deleted file mode 100644 index 2dff376c6..000000000 --- a/import-scripts/test/resources/merge_ddp_files/no_patientid/data_clinical_patient.txt +++ /dev/null @@ -1,14 +0,0 @@ -#Patient Identifier Display name 1 Display name 2 Display name 3 -#Identifier to uniquely specify a patient. Description 1 Description 2 Description 3 -#STRING STRING STRING STRING -#0 1 1 1 -PATIENT_ID PT_CLIN_ATTR_1 PT_CLIN_ATTR_2 PT_CLIN_ATTR_3 -P-000000A VALUE1 VALUE2 VALUE3 -P-000000B VALUE1 VALUE2 VALUE3 -P-000000C VALUE1 VALUE2 VALUE3 -P-000000D VALUE1 VALUE2 VALUE3 -P-000000E VALUE1 VALUE2 VALUE3 -P-000000F VALUE1 VALUE2 VALUE3 -P-000000G VALUE1 VALUE2 VALUE3 -P-000000H VALUE1 VALUE2 VALUE3 -P-000000I VALUE1 VALUE2 VALUE3 \ No newline at end of file diff --git a/import-scripts/test/resources/merge_ddp_files/no_patientid/ddp_naaccr.txt b/import-scripts/test/resources/merge_ddp_files/no_patientid/ddp_naaccr.txt deleted file mode 100644 index d3d64accc..000000000 --- a/import-scripts/test/resources/merge_ddp_files/no_patientid/ddp_naaccr.txt +++ /dev/null @@ -1,6 +0,0 @@ -X NAACCR_SEX_CODE NAACCR_RACE_CODE_PRIMARY NAACCR_RACE_CODE_SECONDARY NAACCR_RACE_CODE_TERTIARY NAACCR_ETHNICITY_CODE BIRTH_YEAR -P-000000A A B C D E F -P-000000B A B C D E F -P-000000C A B C D E F -P-000000D A B C D E F -P-000000E A B C D E F \ No newline at end of file diff --git a/import-scripts/test/test_merge_ddp_files.py b/import-scripts/test/test_merge_ddp_files.py deleted file mode 100644 index 847d91f64..000000000 --- a/import-scripts/test/test_merge_ddp_files.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Provides unit tests for the test_merge_ddp_files.py script functionality. - -To run the tests, execute the following command from the parent -directory of this script: - python -m unittest discover test -""" - -import unittest -import os - -from merge_ddp_files import merge_ddp_files - -class TestMergeDDPFiles(unittest.TestCase): - - # Show the diff even if it is very big - maxDiff = None - - @classmethod - def setUpClass(cls): - cls.base_dir = 'test/resources/merge_ddp_files' - - def test_duplicates(self): - sub_dir = 'duplicates' - ddp_files = [ - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr1.txt'), - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr2.txt'), - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr3.txt'), - ] - self.compare_expected_output_to_actual(sub_dir, ddp_files) - - def test_not_on_masterlist(self): - sub_dir = 'not_on_masterlist' - ddp_files = [ - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr1.txt'), - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr2.txt'), - ] - self.compare_expected_output_to_actual(sub_dir, ddp_files) - - def test_diff_headers(self): - sub_dir = 'diff_headers' - ddp_files = [ - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr.txt'), - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_vital_status.txt'), - ] - self.assertRaises(ValueError, self.compare_expected_output_to_actual, sub_dir, ddp_files) - - def test_no_patientid(self): - sub_dir = 'no_patientid' - ddp_files = [ - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_naaccr.txt'), - ] - self.assertRaises(IndexError, self.compare_expected_output_to_actual, sub_dir, ddp_files) - - def test_has_metadata_headers(self): - sub_dir = 'has_metadata_headers' - ddp_files = [ - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_vital_status1.txt'), - os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'ddp_vital_status2.txt'), - ] - self.compare_expected_output_to_actual(sub_dir, ddp_files) - - def compare_expected_output_to_actual(self, sub_dir, ddp_files): - clinical_file = os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'data_clinical_patient.txt') - output_file = os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'merged.txt') - expected_file = os.path.join(TestMergeDDPFiles.base_dir, sub_dir, 'expected.txt') - - try: - merge_ddp_files(ddp_files, clinical_file, output_file) - except (KeyError, ValueError): - if os.path.exists(output_file): - os.remove(output_file) - raise - - # Read output file and compare it to expected output - with open(expected_file, 'r') as expected_out: - with open(output_file, 'r') as actual_out: - self.assertEqual(expected_out.read(), actual_out.read()) - - # Clean up sample output file - os.remove(output_file) - - -if __name__ == '__main__': - unittest.main() diff --git a/import-scripts/transpose_cna_py3.py b/import-scripts/transpose_cna_py3.py new file mode 100644 index 000000000..e17003e35 --- /dev/null +++ b/import-scripts/transpose_cna_py3.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +""" transpose_cna_py3.py +This script transposes the data_CNA.txt file so that sample IDs are in the first column rather than across the header. +This format was requested to improve ease of anonymizing sample IDs for the Sophia Genetics data. + +Usage: + python3 transpose_cna_py3.py $INPUT_DATA_CNA_FILE_PATH +Example: + python3 transpose_cna_py3.py path/to/data_cna.txt \ +""" + +import argparse +import os +import pandas as pd + + +def transpose_and_write_cna_file(data_cna_file): + # Reads CNA file, skipping comments + df = pd.read_csv(data_cna_file, sep='\t', comment='#') + + # Renames the Hugo_Symbol column so that the column header is correct when transposed + df = df.rename(columns={'Hugo_Symbol': 'SAMPLE_ID'}) + + # Transposes the data frame + df = df.T + + # Writes the transposed data frame to the original file path + df.to_csv(data_cna_file, sep='\t', header=None) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Transpose data_CNA.txt file so that sample IDs are in the first column rather than across the header' + ) + parser.add_argument('data_cna_file', help='Path to location data_CNA.txt file') + args = parser.parse_args() + data_cna_file = args.data_cna_file + + # Ensure that data_CNA.txt file exists + if not os.path.exists(data_cna_file): + raise FileNotFoundError(f'File not found at {data_cna_file}') + + transpose_and_write_cna_file(data_cna_file) diff --git a/import-scripts/update-az-mskimpact.sh b/import-scripts/update-az-mskimpact.sh index 1eccec487..69e755d5c 100755 --- a/import-scripts/update-az-mskimpact.sh +++ b/import-scripts/update-az-mskimpact.sh @@ -1,14 +1,12 @@ #!/usr/bin/env bash export AZ_REPO_NAME="az-data" -export AZ_DATA_HOME="$PORTAL_DATA_HOME/$AZ_REPO_NAME" export AZ_MSKIMPACT_STABLE_ID="az_mskimpact" -export AZ_MSK_IMPACT_DATA_HOME="$AZ_DATA_HOME/$AZ_MSKIMPACT_STABLE_ID" export AZ_TMPDIR=$AZ_DATA_HOME/tmp # Patient and sample attributes that we want to deliver in our data -DELIVERED_PATIENT_ATTRIBUTES="PATIENT_ID PARTC_CONSENTED_12_245 AGE_CURRENT RACE RELIGION SEX ETHNICITY OS_STATUS OS_MONTHS" -DELIVERED_SAMPLE_ATTRIBUTES="SAMPLE_ID PATIENT_ID CANCER_TYPE SAMPLE_TYPE SAMPLE_CLASS METASTATIC_SITE PRIMARY_SITE CANCER_TYPE_DETAILED GENE_PANEL SO_COMMENTS SAMPLE_COVERAGE TUMOR_PURITY ONCOTREE_CODE MSI_COMMENT MSI_SCORE MSI_TYPE SOMATIC_STATUS AGE_AT_SEQ_REPORTED_YEARS ARCHER CVR_TMB_COHORT_PERCENTILE CVR_TMB_SCORE CVR_TMB_TT_COHORT_PERCENTILE" +DELIVERED_PATIENT_ATTRIBUTES="PATIENT_ID PARTC_CONSENTED_12_245 CURRENT_AGE_DEID RACE GENDER ETHNICITY OS_STATUS OS_MONTHS" +DELIVERED_SAMPLE_ATTRIBUTES="SAMPLE_ID PATIENT_ID CANCER_TYPE SAMPLE_TYPE SAMPLE_CLASS METASTATIC_SITE PRIMARY_SITE CANCER_TYPE_DETAILED GENE_PANEL SO_COMMENTS SAMPLE_COVERAGE TUMOR_PURITY ONCOTREE_CODE MSI_COMMENT MSI_SCORE MSI_TYPE SOMATIC_STATUS ARCHER CVR_TMB_COHORT_PERCENTILE CVR_TMB_SCORE CVR_TMB_TT_COHORT_PERCENTILE" # Stores an array of clinical attributes found in the data + attributes we want to filter, respectively unset clinical_attributes_in_file @@ -17,6 +15,7 @@ unset clinical_attributes_to_filter_arg declare -g clinical_attributes_to_filter_arg="unset" source $PORTAL_HOME/scripts/dmp-import-vars-functions.sh +source $PORTAL_HOME/scripts/filter-clinical-arg-functions.sh function report_error() { # Error message provided as an argument @@ -62,7 +61,7 @@ function transfer_to_az_sftp_server() { # Connect and transfer data # With use of here-doc, there must be no leading whitespace until EOF sftp -i "$TRANSFER_KEY" "$SFTP_USER"@"$SERVICE_ENDPOINT" -b <&2 - return 1 - fi - - # Search file for header line - while read -r line ; do - if [ ${#line} -eq 0 ] ; then - echo "error: first uncommented line in $clinical_attribute_filepath was empty" >&2 - return 1 - fi - if ! [ ${line:0:1} == "#" ] ; then - clinical_attribute_header_line=$line - break - fi - done < "$clinical_attribute_filepath" - if [ "$clinical_attribute_header_line" == "unset" ] ; then - echo "error: unable to find header line in $clinical_attribute_filepath" >&2 - return 1 - fi -} - -function find_clinical_attributes_in_file() { - # Path to clinical file taken as an argument - clinical_attribute_filepath=$1 - - # Results (array of clinical attributes) are stored in this global array - clinical_attributes_in_file=() - if ! find_clinical_attribute_header_line_from_file "$clinical_attribute_filepath" ; then - return 1 - fi - for attribute in $clinical_attribute_header_line ; do - clinical_attributes_in_file[$attribute]+=1 - done -} - -function find_clinical_attributes_to_filter_arg() { - # Path to clinical file taken as an argument - clinical_attribute_filepath=$1 - - # Must be either "patient" or "sample" - clinical_attribute_filetype=$2 - - declare -A clinical_attributes_to_filter=() - if ! find_clinical_attributes_in_file "$clinical_attribute_filepath" ; then - return 1 - fi - - # Populate delivered attributes for given file type - unset delivered_attributes - declare -A delivered_attributes=() - case $clinical_attribute_filetype in - patient) - for attribute in $DELIVERED_PATIENT_ATTRIBUTES ; do - delivered_attributes[$attribute]+=1 - done - ;; - sample) - for attribute in $DELIVERED_SAMPLE_ATTRIBUTES ; do - delivered_attributes[$attribute]+=1 - done - ;; - *) - echo "error: illegal filetype passed to find_clinical_attributes_to_filter() : $clinical_attribute_filetype" >&2 - return 1 - ;; - esac - - # Determine which clinical attributes we need to filter based on the attributes found in the file - for attribute in ${!clinical_attributes_in_file[@]} ; do - if [ -z ${delivered_attributes[$attribute]} ] ; then - clinical_attributes_to_filter[$attribute]+=1 - fi - done - - # Put the list attributes we want to filter in a comma separated string - clinical_attributes_to_filter_arg="" - list_size=0 - for attribute in ${!clinical_attributes_to_filter[@]} ; do - clinical_attributes_to_filter_arg="$clinical_attributes_to_filter_arg$attribute" - list_size=$(($list_size+1)) - if [ "$list_size" -lt ${#clinical_attributes_to_filter[@]} ] ; then - clinical_attributes_to_filter_arg="$clinical_attributes_to_filter_arg," - fi - done -} - -function filter_clinical_attribute_columns() { - # Determine which columns to exclude in the patient file +function filter_clinical_cols() { PATIENT_INPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt" PATIENT_OUTPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt.filtered" - find_clinical_attributes_to_filter_arg "$PATIENT_INPUT_FILEPATH" patient - PATIENT_EXCLUDED_HEADER_FIELD_LIST="$clinical_attributes_to_filter_arg" + filter_clinical_attribute_columns "$PATIENT_INPUT_FILEPATH" "$DELIVERED_PATIENT_ATTRIBUTES" "$PATIENT_OUTPUT_FILEPATH" # Determine which columns to exclude in the sample file SAMPLE_INPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt" SAMPLE_OUTPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt.filtered" - find_clinical_attributes_to_filter_arg "$SAMPLE_INPUT_FILEPATH" sample - SAMPLE_EXCLUDED_HEADER_FIELD_LIST="$clinical_attributes_to_filter_arg" - - # Filter out the columns we want to exclude in both files - $PYTHON_BINARY $PORTAL_HOME/scripts/filter_clinical_data.py -c "$PATIENT_INPUT_FILEPATH" -e "$PATIENT_EXCLUDED_HEADER_FIELD_LIST" > "$PATIENT_OUTPUT_FILEPATH" && - $PYTHON_BINARY $PORTAL_HOME/scripts/filter_clinical_data.py -c "$SAMPLE_INPUT_FILEPATH" -e "$SAMPLE_EXCLUDED_HEADER_FIELD_LIST" > "$SAMPLE_OUTPUT_FILEPATH" && - - # Rewrite the patient and sample files with updated data - mv "$PATIENT_OUTPUT_FILEPATH" "$PATIENT_INPUT_FILEPATH" && - mv "$SAMPLE_OUTPUT_FILEPATH" "$SAMPLE_INPUT_FILEPATH" + filter_clinical_attribute_columns "$SAMPLE_INPUT_FILEPATH" "$DELIVERED_SAMPLE_ATTRIBUTES" "$SAMPLE_OUTPUT_FILEPATH" +} + +function rename_cdm_clinical_attribute_columns() { + # Rename clinical patient attributes coming from CDM: + # CURRENT_AGE_DEID -> AGE_CURRENT + # GENDER -> SEX + + PATIENT_INPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt" + PATIENT_OUTPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt.renamed" + + sed -e '1s/CURRENT_AGE_DEID/AGE_CURRENT/' -e '1s/GENDER/SEX/' $PATIENT_INPUT_FILEPATH > $PATIENT_OUTPUT_FILEPATH && + mv "$PATIENT_OUTPUT_FILEPATH" "$PATIENT_INPUT_FILEPATH" } function add_metadata_headers() { @@ -286,6 +191,22 @@ function standardize_mutations_data() { $PYTHON_BINARY $PORTAL_HOME/scripts/standardize_mutations_data.py -f "$NSOUT_MUTATIONS_INPUT_FILEPATH" } +function remove_duplicate_maf_variants() { + MUTATIONS_EXTD_INPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt" + NSOUT_MUTATIONS_INPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_nonsignedout_mutations.txt" + + MUTATIONS_EXTD_OUTPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_mutations_extended_merged.txt" + NSOUT_MUTATIONS_OUTPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_nonsignedout_mutations_merged.txt" + + # Remove duplicate variants from MAF files + $PYTHON_BINARY $PORTAL_HOME/scripts/remove-duplicate-maf-variants.py -i "$MUTATIONS_EXTD_INPUT_FILEPATH" -o "$MUTATIONS_EXTD_OUTPUT_FILEPATH" && + $PYTHON_BINARY $PORTAL_HOME/scripts/remove-duplicate-maf-variants.py -i "$NSOUT_MUTATIONS_INPUT_FILEPATH" -o "$NSOUT_MUTATIONS_OUTPUT_FILEPATH" && + + # Rewrite mutation files with updated data + mv "$MUTATIONS_EXTD_OUTPUT_FILEPATH" "$MUTATIONS_EXTD_INPUT_FILEPATH" && + mv "$NSOUT_MUTATIONS_OUTPUT_FILEPATH" "$NSOUT_MUTATIONS_INPUT_FILEPATH" +} + function standardize_structural_variant_data() { DATA_SV_INPUT_FILEPATH="$AZ_MSK_IMPACT_DATA_HOME/data_sv.txt" $PYTHON_BINARY $PORTAL_HOME/scripts/standardize_structural_variant_data.py -f "$DATA_SV_INPUT_FILEPATH" @@ -319,7 +240,7 @@ function generate_case_lists() { } # ------------------------------------------------------------------------------------------------------------------------ -# 1. Pull latest from AstraZeneca repo (az-data) +# Pull latest from AstraZeneca repo (az-data) printTimeStampedDataProcessingStepMessage "Pull of AstraZeneca MSK-IMPACT data updates" if ! pull_latest_data_from_az_git_repo ; then @@ -327,7 +248,7 @@ if ! pull_latest_data_from_az_git_repo ; then fi # ------------------------------------------------------------------------------------------------------------------------ -# 2. Copy data from local clone of MSK Solid Heme repo to local clone of AZ repo +# Copy data from local clone of MSK Solid Heme repo to local clone of AZ repo # Create temporary directory to store data before subsetting if ! [ -d "$AZ_TMPDIR" ] ; then @@ -346,7 +267,7 @@ if [ $? -gt 0 ] ; then fi # ------------------------------------------------------------------------------------------------------------------------ -# 3. Post-process the dataset +# Post-process the dataset printTimeStampedDataProcessingStepMessage "Subset and merge of Part A Consented patients for AstraZeneca MSK-IMPACT" @@ -381,10 +302,14 @@ if ! rename_files_in_delivery_directory ; then fi # Filter clinical attribute columns from clinical files -if ! filter_clinical_attribute_columns ; then +if ! filter_clinical_cols ; then report_error "ERROR: Failed to filter non-delivered clinical attribute columns for AstraZeneca MSK-IMPACT. Exiting." fi +if ! rename_cdm_clinical_attribute_columns ; then + report_error "ERROR: Failed to rename CDM clinical attribute columns for AstraZeneca MSK-IMPACT. Exiting." +fi + # Add metadata headers to clinical files if ! add_metadata_headers ; then report_error "ERROR: Failed to add metadata headers to clinical attribute files for AstraZeneca MSK-IMPACT. Exiting." @@ -405,15 +330,20 @@ if ! standardize_mutations_data ; then report_error "ERROR: Failed to standardize mutations files for AstraZeneca MSK-IMPACT. Exiting." fi +# Remove duplicate variants from MAF files +if ! remove_duplicate_maf_variants ; then + report_error "ERROR: Failed to remove duplicate variants from MAF files for AstraZeneca MSK-IMPACT. Exiting." +fi + # Standardize structural variant data by removing records with invalid genes and standardizing the file header if ! standardize_structural_variant_data ; then report_error "ERROR: Failed to standardize structural variant data for AstraZeneca MSK-IMPACT. Exiting." fi # Anonymize ages -if ! anonymize_age_at_seq_with_cap ; then - report_error "ERROR: Failed to anonymize AGE_AT_SEQUENCING_REPORTED_YEARS for AstraZeneca MSK-IMPACT. Exiting." -fi +#if ! anonymize_age_at_seq_with_cap ; then +# report_error "ERROR: Failed to anonymize AGE_AT_SEQUENCING_REPORTED_YEARS for AstraZeneca MSK-IMPACT. Exiting." +#fi printTimeStampedDataProcessingStepMessage "Filter non-delivered files and include delivered meta files for AstraZeneca MSK-IMPACT" @@ -428,7 +358,7 @@ if [[ -d "$AZ_TMPDIR" && "$AZ_TMPDIR" != "/" ]] ; then fi # ------------------------------------------------------------------------------------------------------------------------ -# 4. Run changelog script +# Run changelog script printTimeStampedDataProcessingStepMessage "Generate changelog for AstraZeneca MSK-IMPACT" $PYTHON3_BINARY $PORTAL_HOME/scripts/generate_az_study_changelog_py3.py $AZ_MSK_IMPACT_DATA_HOME @@ -438,7 +368,7 @@ if [ $? -gt 0 ] ; then fi # ------------------------------------------------------------------------------------------------------------------------ -# 5. Filter germline events from mutation file and structural variant file +# Filter germline events from mutation file and structural variant file printTimeStampedDataProcessingStepMessage "Filter germline events for AstraZeneca MSK-IMPACT" mutation_filepath="$AZ_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt" @@ -467,7 +397,7 @@ fi mv $sv_filtered_filepath $sv_filepath # ------------------------------------------------------------------------------------------------------------------------ -# 6. Generate case list files +# Generate case list files printTimeStampedDataProcessingStepMessage "Generate case list files for AstraZeneca MSK-IMPACT" if ! generate_case_lists ; then @@ -475,7 +405,7 @@ if ! generate_case_lists ; then fi # ------------------------------------------------------------------------------------------------------------------------ -# 7. Push the updated data to GitHub +# Push the updated data to GitHub printTimeStampedDataProcessingStepMessage "Push data updates to AstraZeneca MSK-IMPACT git repository" if ! push_updates_to_az_git_repo ; then @@ -483,7 +413,7 @@ if ! push_updates_to_az_git_repo ; then fi # ------------------------------------------------------------------------------------------------------------------------ -# 7. Push the updated data to AstraZeneca's SFTP server +# Push the updated data to AstraZeneca's SFTP server printTimeStampedDataProcessingStepMessage "Transfer data updates to SFTP server for AstraZeneca MSK-IMPACT" @@ -491,5 +421,11 @@ if ! transfer_to_az_sftp_server ; then report_error "ERROR: Failed to transfer data updates to SFTP server for AstraZeneca MSK-IMPACT. Exiting." fi +# ------------------------------------------------------------------------------------------------------------------------ +# Cleanup AZ git repo + +printTimeStampedDataProcessingStepMessage "Cleaning up untracked files from AZ repo" +bash $PORTAL_HOME/scripts/datasource-repo-cleanup.sh $AZ_DATA_HOME + # Send a message on success sendImportSuccessMessageMskPipelineLogsSlack "ASTRAZENECA MSKIMPACT" diff --git a/import-scripts/update-sophia-mskimpact.sh b/import-scripts/update-sophia-mskimpact.sh new file mode 100755 index 000000000..077969435 --- /dev/null +++ b/import-scripts/update-sophia-mskimpact.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash + +# File containing list of patients should be passed in as argument +export SUBSET_FILE="$1" +export COHORT_NAME="$2" +export CURRENT_DATE="$(date '+%m.%d.%y')" + +export SOPHIA_MSK_IMPACT_DATA_HOME="$SOPHIA_DATA_HOME/$COHORT_NAME" +export SOPHIA_TMPDIR="$SOPHIA_MSK_IMPACT_DATA_HOME/tmp" + +# Patient and sample attributes that we want to deliver in our data +DELIVERED_PATIENT_ATTRIBUTES="PATIENT_ID PARTC_CONSENTED_12_245 RACE SEX ETHNICITY" +DELIVERED_SAMPLE_ATTRIBUTES="SAMPLE_ID PATIENT_ID CANCER_TYPE SAMPLE_TYPE SAMPLE_CLASS METASTATIC_SITE PRIMARY_SITE CANCER_TYPE_DETAILED GENE_PANEL SAMPLE_COVERAGE TUMOR_PURITY ONCOTREE_CODE MSI_SCORE MSI_TYPE SOMATIC_STATUS ARCHER CVR_TMB_COHORT_PERCENTILE CVR_TMB_SCORE CVR_TMB_TT_COHORT_PERCENTILE SEQ_DATE" + +# Duplicate columns that we want to filter out of MAF files +MUTATIONS_EXTENDED_COLS_TO_FILTER="amino_acid_change,cdna_change,transcript,COMMENTS,Comments,comments,Matched_Norm_Sample_Barcode" +NONSIGNEDOUT_COLS_TO_FILTER="amino_acid_change,cdna_change,transcript,Comments,comments,Matched_Norm_Sample_Barcode" + +# Stores an array of clinical attributes found in the data + attributes we want to filter, respectively +unset clinical_attributes_in_file +declare -gA clinical_attributes_in_file=() +unset clinical_attributes_to_filter_arg +declare -g clinical_attributes_to_filter_arg="unset" + +source $PORTAL_HOME/scripts/dmp-import-vars-functions.sh +source $PORTAL_HOME/scripts/filter-clinical-arg-functions.sh + +function report_error() { + # Error message provided as an argument + error_message="$1" + echo -e "$error_message" + exit 1 +} + +function pull_latest_data_from_sophia_git_repo() { + ( # Executed in a subshell to avoid changing the actual working directory + # If any statement fails, the return value of the entire expression is the failure status + cd $SOPHIA_DATA_HOME && + $GIT_BINARY fetch && + $GIT_BINARY reset origin/main --hard && + $GIT_BINARY lfs pull && + $GIT_BINARY clean -f -d + ) +} + +function push_updates_to_sophia_git_repo() { + ( # Executed in a subshell to avoid changing the actual working directory + # If any statement fails, the return value of the entire expression is the failure status + cd $SOPHIA_DATA_HOME && + $GIT_BINARY add * && + $GIT_BINARY commit -m "$COHORT_NAME data $CURRENT_DATE" && + $GIT_BINARY push origin + ) +} + +function filter_files_in_delivery_directory() { + unset filenames_to_deliver + declare -A filenames_to_deliver + + # Data files to deliver + filenames_to_deliver[data_clinical_patient.txt]+=1 + filenames_to_deliver[data_clinical_sample.txt]+=1 + filenames_to_deliver[data_CNA.txt]+=1 + filenames_to_deliver[data_gene_matrix.txt]+=1 + filenames_to_deliver[data_mutations_extended.txt]+=1 + filenames_to_deliver[data_mutations_non_signedout.txt]+=1 + filenames_to_deliver[data_sv.txt]+=1 + filenames_to_deliver[sophia_mskimpact_data_cna_hg19.seg]+=1 + filenames_to_deliver[DMP_IDs.txt]+=1 + + # Remove any files/directories that are not specified above + for filepath in $SOPHIA_MSK_IMPACT_DATA_HOME/* ; do + filename=$(basename $filepath) + if [ -z ${filenames_to_deliver[$filename]} ] ; then + if ! rm -rf $filepath ; then + return 1 + fi + fi + done + return 0 +} + +function rename_files_in_delivery_directory() { + # We want to rename: + # az_mskimpact_data_cna_hg19.seg -> sophia_mskimpact_data_cna_hg19.seg + # data_nonsignedout_mutations.txt -> data_mutations_non_signedout.txt + + unset filenames_to_rename + declare -A filenames_to_rename + + # Data files to rename + filenames_to_rename[az_mskimpact_data_cna_hg19.seg]=sophia_mskimpact_data_cna_hg19.seg + filenames_to_rename[data_nonsignedout_mutations.txt]=data_mutations_non_signedout.txt + + for original_filename in "${!filenames_to_rename[@]}" + do + if [ -f "$SOPHIA_MSK_IMPACT_DATA_HOME/$original_filename" ]; then + if ! mv "$SOPHIA_MSK_IMPACT_DATA_HOME/$original_filename" "$SOPHIA_MSK_IMPACT_DATA_HOME/${filenames_to_rename[$original_filename]}" ; then + return 1 + fi + fi + done + return 0 +} + +function add_seq_date_to_sample_file() { + SEQ_DATE_FILENAME="cvr/seq_date.txt" + MSK_ACCESS_SEQ_DATE="$MSK_ACCESS_DATA_HOME/$SEQ_DATE_FILENAME" + MSK_HEMEPACT_SEQ_DATE="$MSK_HEMEPACT_DATA_HOME/$SEQ_DATE_FILENAME" + MSK_IMPACT_SEQ_DATE="$MSK_IMPACT_DATA_HOME/$SEQ_DATE_FILENAME" + MERGED_SEQ_DATE="./merged_seq_date.txt" + + SAMPLE_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt" + SAMPLE_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt.filtered" + KEY_COLUMNS="SAMPLE_ID PATIENT_ID" + + $PYTHON3_BINARY $PORTAL_HOME/scripts/combine_files_py3.py -i "$MSK_ACCESS_SEQ_DATE" "$MSK_HEMEPACT_SEQ_DATE" "$MSK_IMPACT_SEQ_DATE" -o "$MERGED_SEQ_DATE" -m outer && + $PYTHON3_BINARY $PORTAL_HOME/scripts/combine_files_py3.py -i "$SAMPLE_INPUT_FILEPATH" "$MERGED_SEQ_DATE" -o "$SAMPLE_OUTPUT_FILEPATH" -c $KEY_COLUMNS -m left && + + mv "$SAMPLE_OUTPUT_FILEPATH" "$SAMPLE_INPUT_FILEPATH" + + # Metadata headers will be added back in a later function call +} + +function filter_clinical_cols() { + PATIENT_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt" + PATIENT_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt.filtered" + filter_clinical_attribute_columns "$PATIENT_INPUT_FILEPATH" "$DELIVERED_PATIENT_ATTRIBUTES" "$PATIENT_OUTPUT_FILEPATH" + + # Determine which columns to exclude in the sample file + SAMPLE_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt" + SAMPLE_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt.filtered" + filter_clinical_attribute_columns "$SAMPLE_INPUT_FILEPATH" "$DELIVERED_SAMPLE_ATTRIBUTES" "$SAMPLE_OUTPUT_FILEPATH" +} + +function filter_replicated_maf_columns() { + MUTATIONS_EXTENDED_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt" + MUTATIONS_EXTENDED_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt.filtered" + MUTATIONS_NONSIGNEDOUT_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_non_signedout.txt" + MUTATIONS_NONSIGNEDOUT_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_non_signedout.txt.filtered" + + # Filter out the columns we want to exclude in both files + $PYTHON_BINARY $PORTAL_HOME/scripts/filter_clinical_data.py -c "$MUTATIONS_EXTENDED_INPUT_FILEPATH" -e "$MUTATIONS_EXTENDED_COLS_TO_FILTER" > "$MUTATIONS_EXTENDED_OUTPUT_FILEPATH" && + $PYTHON_BINARY $PORTAL_HOME/scripts/filter_clinical_data.py -c "$MUTATIONS_NONSIGNEDOUT_INPUT_FILEPATH" -e "$NONSIGNEDOUT_COLS_TO_FILTER" > "$MUTATIONS_NONSIGNEDOUT_OUTPUT_FILEPATH" && + + # Rewrite the MAF files + mv "$MUTATIONS_EXTENDED_OUTPUT_FILEPATH" "$MUTATIONS_EXTENDED_INPUT_FILEPATH" && + mv "$MUTATIONS_NONSIGNEDOUT_OUTPUT_FILEPATH" "$MUTATIONS_NONSIGNEDOUT_INPUT_FILEPATH" +} + +function remove_duplicate_maf_variants() { + MUTATIONS_EXTD_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt" + NSOUT_MUTATIONS_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_non_signedout.txt" + + MUTATIONS_EXTD_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_extended_merged.txt" + NSOUT_MUTATIONS_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_non_signedout_merged.txt" + + # Remove duplicate variants from MAF files + # CVR data can contain duplicates for a gene and its alias + $PYTHON_BINARY $PORTAL_HOME/scripts/remove-duplicate-maf-variants.py -i "$MUTATIONS_EXTD_INPUT_FILEPATH" -o "$MUTATIONS_EXTD_OUTPUT_FILEPATH" && + $PYTHON_BINARY $PORTAL_HOME/scripts/remove-duplicate-maf-variants.py -i "$NSOUT_MUTATIONS_INPUT_FILEPATH" -o "$NSOUT_MUTATIONS_OUTPUT_FILEPATH" && + + # Rewrite mutation files with updated data + mv "$MUTATIONS_EXTD_OUTPUT_FILEPATH" "$MUTATIONS_EXTD_INPUT_FILEPATH" && + mv "$NSOUT_MUTATIONS_OUTPUT_FILEPATH" "$NSOUT_MUTATIONS_INPUT_FILEPATH" +} + +function add_metadata_headers() { + # Calling merge.py strips out metadata headers from our clinical files - add them back in + CDD_URL="http://cdd.cbioportal.mskcc.org/api/" + INPUT_FILENAMES="$SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_sample.txt $SOPHIA_MSK_IMPACT_DATA_HOME/data_clinical_patient.txt" + $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -f $INPUT_FILENAMES -c "$CDD_URL" -s mskimpact +} + +function transpose_cna_data() { + # Transpose the CNA file so that sample IDs are contained in the first column instead of the header + DATA_CNA_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_CNA.txt" + $PYTHON3_BINARY $PORTAL_HOME/scripts/transpose_cna_py3.py "$DATA_CNA_INPUT_FILEPATH" +} + +function remove_sequenced_samples_header() { + MAF_INPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt" + MAF_OUTPUT_FILEPATH="$SOPHIA_MSK_IMPACT_DATA_HOME/data_mutations_extended.txt.tmp" + + # This removes the sequenced_samples header from the MAF + awk '!/^#sequenced_samples:/' "$MAF_INPUT_FILEPATH" > "$MAF_OUTPUT_FILEPATH" && mv "$MAF_OUTPUT_FILEPATH" "$MAF_INPUT_FILEPATH" +} + +# Pull latest from Sophia repo (sophia-data) +printTimeStampedDataProcessingStepMessage "Pull of Sophia MSK-IMPACT data updates" +if ! pull_latest_data_from_sophia_git_repo ; then + report_error "ERROR: Failed git pull for Sophia MSK-IMPACT. Exiting." +fi + +# Create temporary directory to store data before subsetting +if ! [ -d "$SOPHIA_TMPDIR" ] ; then + if ! mkdir -p "$SOPHIA_TMPDIR" ; then + report_error "ERROR: Failed to create temporary directory for Sophia MSK-IMPACT. Exiting." + fi +fi +if [[ -d "$SOPHIA_TMPDIR" && "$SOPHIA_TMPDIR" != "/" ]] ; then + rm -rf "$SOPHIA_TMPDIR"/* +fi + +# Copy data from local clone of AZ repo to Sophia directory +cp -a $AZ_MSK_IMPACT_DATA_HOME/* $SOPHIA_TMPDIR + +if [ $? -gt 0 ] ; then + report_error "ERROR: Failed to populate temporary directory for Sophia MSK-IMPACT. Exiting." +fi + +# Post-process the dataset +printTimeStampedDataProcessingStepMessage "Subset Sophia MSK-IMPACT" + +# Write out the subsetted data +$PYTHON_BINARY $PORTAL_HOME/scripts/merge.py \ + --study-id="az_mskimpact" \ + --subset="$SUBSET_FILE" \ + --output-directory="$SOPHIA_MSK_IMPACT_DATA_HOME" \ + --merge-clinical="true" \ + $SOPHIA_TMPDIR + +if [ $? -gt 0 ] ; then + report_error "ERROR: Failed to write out subsetted data for Sophia MSK-IMPACT. Exiting." +fi + +# Rename files that need to be renamed +if ! rename_files_in_delivery_directory ; then + report_error "ERROR: Failed to rename files for Sophia MSK-IMPACT. Exiting." +fi + +# Transpose CNA file +if ! transpose_cna_data ; then + report_error "ERROR: Failed to transpose CNA file for Sophia MSK-IMPACT. Exiting." +fi + +# Remove sequenced_samples header from MAF +if ! remove_sequenced_samples_header ; then + report_error "ERROR: Failed to remove sequenced_samples header from MAF for Sophia MSK-IMPACT. Exiting." +fi + +# Add SEQ_DATE to clinical sample file +if ! add_seq_date_to_sample_file; then + report_error "ERROR: Failed to add SEQ_DATE column to clinical sample file for Sophia MSK-IMPACT. Exiting." +fi + +# Filter clinical attribute columns from clinical files +if ! filter_clinical_cols ; then + report_error "ERROR: Failed to filter non-delivered clinical attribute columns for Sophia MSK-IMPACT. Exiting." +fi + +# Filter replicated columns from MAF files +if ! filter_replicated_maf_columns ; then + report_error "ERROR: Failed to filter duplicated columns in MAF files for Sophia MSK-IMPACT. Exiting." +fi + +# Remove duplicate variants from MAF files +if ! remove_duplicate_maf_variants ; then + report_error "ERROR: Failed to remove duplicate variants from MAF files for Sophia MSK-IMPACT. Exiting." +fi + +# Add metadata headers to clinical files +if ! add_metadata_headers ; then + report_error "ERROR: Failed to add metadata headers to clinical attribute files for Sophia MSK-IMPACT. Exiting." +fi + +# Filter out files which are not delivered +if ! filter_files_in_delivery_directory ; then + report_error "ERROR: Failed to filter non-delivered files for Sophia MSK-IMPACT. Exiting." +fi + +# Remove temporary directory now that the subset has been merged and post-processed +if [[ -d "$SOPHIA_TMPDIR" && "$SOPHIA_TMPDIR" != "/" ]] ; then + rm -rf "$SOPHIA_TMPDIR" +fi + +# Zip data files for easier data transfer +zip "$SOPHIA_MSK_IMPACT_DATA_HOME/sophia-$COHORT_NAME-data-$CURRENT_DATE.zip" $SOPHIA_MSK_IMPACT_DATA_HOME/*data* + +# Push updated data to GitHub +if ! push_updates_to_sophia_git_repo ; then + report_error "ERROR: Failed git push for AstraZeneca MSK-IMPACT. Exiting." +fi + +printTimeStampedDataProcessingStepMessage "Cleaning up untracked files from Sophia repo" +bash $PORTAL_HOME/scripts/datasource-repo-cleanup.sh $SOPHIA_DATA_HOME \ No newline at end of file diff --git a/import-scripts/update_cfdna_clinical_sample_patient_ids_via_dremio.sh b/import-scripts/update_cfdna_clinical_sample_patient_ids_via_dremio.sh new file mode 100755 index 000000000..7810266a7 --- /dev/null +++ b/import-scripts/update_cfdna_clinical_sample_patient_ids_via_dremio.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# +# replace patient ids present in a clnical file in the cmo-access repository +# with the perferred patient id based on a input provided id mapping table. + +# global variables +TAB=$'\t' +PATIENT_ID_MAPPING_FILTERED_FILENAME="patient_id_mapping_filtered.txt" +PATIENT_ID_MAPPING_AMBIGUOUS_FILENAME="patient_id_mapping_ambiguous.txt" +unset input_clinical_filepath +unset output_clinical_dirpath +unset patient_id_mapping_filepath +unset patient_id_mapping_filtered_filepath +unset patient_id_mapping_ambiguous_filepath +unset PROGRAM_NAME +input_clinical_filepath="" +output_clinical_filepath="" +patient_id_mapping_filepath="" +patient_id_mapping_filtered_filepath="" +patient_id_mapping_ambiguous_filepath="" +PROGRAM_NAME="$0" + +function usage() { + echo "Usage: $PROGRAM_NAME input_clinical_file output_directory patient_id_mapping_file" >&2 +} + +# validate and parse arguments +if [[ ${#@} -eq 1 ]] ; then + if [[ "$1" == "-h" || "$1" == "--help" ]] ; then + usage + exit 0 + fi +fi +if [[ ${#@} -ne 3 ]] ; then + usage + exit 1 +fi +input_clinical_filepath="$1" +output_clinical_dirpath="$2" +patient_id_mapping_filepath="$3" +if ! [[ -r "$input_clinical_filepath" ]] ; then + echo "Error: expected a readable input file as input_clinical_file. Unable to read file '$input_clinical_filepath'" >&2 + exit 1 +fi +if ! [[ -w "$output_clinical_dirpath" && -d "$output_clinical_dirpath" ]] ; then + echo "Error: expected a writable output directory as output_directory. Unable to write to '$input_clinical_filepath'" >&2 + exit 1 +fi +if ! [[ -r "$patient_id_mapping_filepath" ]] ; then + echo "Error: expected a readable input file as patient_id_mapping_file. Unable to read file '$patient_id_mapping_filepath'" >&2 + exit 1 +fi +patient_id_mapping_filtered_filepath="$output_clinical_dirpath/$PATIENT_ID_MAPPING_FILTERED_FILENAME" +patient_id_mapping_ambiguous_filepath="$output_clinical_dirpath/$PATIENT_ID_MAPPING_AMBIGUOUS_FILENAME" +output_clinical_filename="$(basename $input_clinical_filepath).updated" +output_clinical_filepath="$output_clinical_dirpath/$output_clinical_filename" + +# filter patient id mappings of all rows missing either dmp-id or cmo-id +if ! cat "$patient_id_mapping_filepath" | egrep -v "^[[:space:]]" | egrep -v "[[:space:]]$" > "$patient_id_mapping_filtered_filepath" ; then + echo "could not filter $patient_id_mapping_filepath of rows missing a dmp id" >&2 + exit 1 +fi + +# generate list of ambiguous cmo patient ids (have multiple associated dmp ids) +if ! cat "$patient_id_mapping_filtered_filepath" | sort | uniq | cut -f2 | sort | uniq -d > "$patient_id_mapping_ambiguous_filepath" ; then + echo "could not find and output ambiguous cases from file $patient_id_mapping_filtered_filepath" >&2 + exit 1 +fi + +# read in the mapping of cmo patient ids to dmp patient ids and store in associative array +unset ambiguous_cmo_patient_id +declare -A ambiguous_cmo_patient_id +while read -r cmo_patient_id ; do + ambiguous_cmo_patient_id[$cmo_patient_id]=1 +done < "$patient_id_mapping_ambiguous_filepath" +unset dmp_id_for_cmo_id +declare -A dmp_id_for_cmo_id +while IFS="" read -r line ; do + line_regex="^([^$TAB][^$TAB]*)$TAB([^$TAB][^$TAB]*)\$" + if ! [[ "$line" =~ $line_regex ]] ; then + echo "malformatted line in $patient_id_mapping_filtered_filepath : $line" >&2 + exit 1 + fi + dmp_patient_id=${BASH_REMATCH[1]} + cmo_patient_id=${BASH_REMATCH[2]} + if ! [ -z ${ambiguous_cmo_patient_id["$cmo_patient_id"]} ] ; then + continue; # skip ambiguous cmo_patient_ids + fi + dmp_id_for_cmo_id["$cmo_patient_id"]="$dmp_patient_id" +done < "$patient_id_mapping_filtered_filepath" + +#replace any cmo-patient-ids in clinical sample file with associated dmp-patient-id +unset headerline +unset patient_id_colnum +while IFS="" read -r line ; do + header_regex="^#" + if [[ "$line" =~ $header_regex ]] ; then + # simply output metadata headers + echo "$line" + continue + fi + if [ -z "$headerline" ] ; then + # set the header + headerline="$line" + echo "$headerline" + # find the column number of PATIENT_ID + colnum=1 + scanheader="$headerline" + while ! [ -z "$scanheader" ] ; do + line_regex="^([^$TAB][^$TAB]*)$TAB(.*)\$" + if ! [[ "$scanheader" =~ $line_regex ]] ; then + # no tab found - examine final column + if [[ "$scanheader" == "PATIENT_ID" ]] ; then + patient_id_colnum=$colnum + scanheader="" # scan is done + fi + break + fi + this_header="${BASH_REMATCH[1]}" + rest_of_line="${BASH_REMATCH[2]}" + if [[ "$this_header" == "PATIENT_ID" ]] ; then + patient_id_colnum=$colnum + scanheader="" # scan is done + else + colnum=$(($colnum+1)) + scanheader="$rest_of_line" + fi + done + if [ -z $patient_id_colnum ] ; then + echo "could not find column header PATIENT_ID in file $input_clinical_filepath" >&2 + exit 1 + fi + continue + fi + # process and output each data line + rest_of_line="$line" + colnum=1 + while true ; do + line_regex="^([^$TAB]*)$TAB(.*)\$" + if ! [[ "$rest_of_line" =~ $line_regex ]] ; then + # no tab found - output final column + if [[ $colnum -eq $patient_id_colnum ]] ; then + patient_id="$rest_of_line" + replacement_patient_id="${dmp_id_for_cmo_id[$patient_id]}" + if ! [ -z $replacement_patient_id ] ; then + echo "$replacement_patient_id" + else + echo "$patient_id" + fi + else + echo "$rest_of_line" + fi + break + fi + this_value="${BASH_REMATCH[1]}" + rest_of_line="${BASH_REMATCH[2]}" + if [[ $colnum -eq $patient_id_colnum ]] ; then + replacement_patient_id="${dmp_id_for_cmo_id[$this_value]}" + if ! [ -z $replacement_patient_id ] ; then + echo -n "$replacement_patient_id$TAB" + else + echo -n "$this_value$TAB" + fi + else + echo -n "$this_value$TAB" + fi + colnum=$(($colnum+1)) + done +done < "$input_clinical_filepath" > "$output_clinical_filepath" diff --git a/integration-tests/fetch_and_compare_to_redcap_schema.py b/integration-tests/fetch_and_compare_to_redcap_schema.py index e069e4b00..e66896500 100644 --- a/integration-tests/fetch_and_compare_to_redcap_schema.py +++ b/integration-tests/fetch_and_compare_to_redcap_schema.py @@ -12,10 +12,10 @@ # Script for integration testing of a pull request # Determines which datasource schema to test based on tags on pull request -# Will fetch and export from redcap accordingly (i.e fetch from darwin and export all darwin based projects) +# Will fetch and export from redcap accordingly # Compares headers between datasource fetch and all related redcap projects # Different headers imply mismatched data schema and returns false -# lib directory holding all jars must be provided (redcap_pipelines, ddp, cvr, crdb, darwin) +# lib directory holding all jars must be provided (redcap_pipelines, ddp, cvr, crdb) # seperate directories to hold fetched files and redcap exports must be provided # --------------------------------------------- @@ -31,15 +31,14 @@ LABEL_TO_TEST_MAPPING = { # '984868388' : "crdb_fetcher", # '984867863' : "cvr_fetcher", -# '984872074' : "darwin_fetcher", '984872969' : "ddp_fetcher" } # generates a mapping for fetchers to fetched and redcap-exported filenames # used for determining which files need to be compared # redcap_project_to_file_mapping: -# { "darwin_fetcher" : { -# "/path/to/fetch/" : ["/path/to/redcap_export/", "/path/to/redcap_export/", ...], +# { "fetcher_filename" : { +# "/path/to/fetch/" : ["/path/to/redcap_export/", "/path/to/redcap_export/", ...], # "" " ["", ...] # } # } @@ -112,10 +111,6 @@ def export_redcap_projects(fetchers_to_test, fetched_file_to_redcap_file_mapping def crdb_fetch(lib, fetch_directory): return "java -jar " + os.path.join(lib, "crdb_fetcher.jar") + " -d " + fetch_directory -# darwin fetch currently does not work because jenkins machine cannot access darwin -def darwin_fetch(lib, fetch_directory): - return "java -jar " + os.path.join(lib, "darwin_fetcher.jar") + " -d " + fetch_directory + " -s mskimpact_heme -c 0" - def cvr_fetch(lib, fetch_directory, truststore, truststore_password): redcap_request = "java -Djavax.net.ssl.trustStore=" + truststore + " -Djavax.net.ssl.trustStorePassword=" + truststore_password + " -jar " + os.path.join(lib, "redcap_pipeline.jar") + " -e -r -p hemepact_data_clinical -d " + fetch_directory redcap_status = subprocess.call(redcap_request, shell = True) @@ -143,7 +138,6 @@ def fetch_data_source_files(fetchers_to_test, fetch_directory, lib, truststore, datasource_fetches = { "crdb_fetcher" : [crdb_fetch], "cvr_fetcher" : [cvr_fetch], - "darwin_fetcher" : [darwin_fetch], "ddp_fetcher" : [ddp_fetch, ddp_pediatrics_fetch] } for fetcher in fetchers_to_test: diff --git a/pom.xml b/pom.xml index 3f96adc97..47ba5c456 100644 --- a/pom.xml +++ b/pom.xml @@ -4,18 +4,15 @@ master pom MSKCC CMO Pipelines Master - 0.1.0 + 1.0.0 master maven module https://github.com/cBioPortal/cmo-pipelines/ common - gdd crdb - darwin redcap cvr - gene ddp import-scripts @@ -24,16 +21,18 @@ org.springframework.boot spring-boot-starter-parent - 2.3.3.RELEASE + 3.1.4 - 5.2.6.RELEASE - 2.11.2 + 6.0.12 + 3.1.4 + 5.0.3 + 2.14.3 1.7.30 3.11.0 - 11 - 11 + 21 + 21 @@ -41,6 +40,7 @@ org.springframework.boot spring-boot-starter-batch + ${spring.boot.version} org.slf4j @@ -67,7 +67,7 @@ junit junit - 4.13 + 4.13.2 test @@ -97,7 +97,7 @@ org.xerial sqlite-jdbc - 3.15.1 + 3.15.1 diff --git a/redcap/pom.xml b/redcap/pom.xml index 77b33e49b..85ee0bf4c 100644 --- a/redcap/pom.xml +++ b/redcap/pom.xml @@ -4,12 +4,12 @@ MSKCC CMO Redcap Pipeline Master Redcap Pipeline redcap - 0.1.0 + 1.0.0 org.mskcc.cmo.ks.redcap org.mskcc.cmo.ks master - 0.1.0 + 1.0.0 pom @@ -22,7 +22,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.3 + 3.5.1 redcap-pipeline-shade-jar diff --git a/redcap/redcap_pipeline/pom.xml b/redcap/redcap_pipeline/pom.xml index 9f46eedc1..ee3c4a01e 100644 --- a/redcap/redcap_pipeline/pom.xml +++ b/redcap/redcap_pipeline/pom.xml @@ -3,23 +3,18 @@ MSKCC CMO Redcap Pipeline Redcap Pipeline redcap_pipeline - 0.1.0 + 1.0.0 org.mskcc.cmo.ks.redcap.pipeline org.mskcc.cmo.ks.redcap redcap - 0.1.0 + 1.0.0 - - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 - org.mskcc.cmo.ks.redcap.source redcap_source - 0.1.0 + 1.0.0 jar diff --git a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/BatchConfiguration.java b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/BatchConfiguration.java index b196767a1..8ea39272e 100644 --- a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/BatchConfiguration.java +++ b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/BatchConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Memorial Sloan-Kettering Cancer Center. + * Copyright (c) 2016, 2024 Memorial Sloan-Kettering Cancer Center. * * This library is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS @@ -35,21 +35,28 @@ import java.util.*; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.springframework.batch.core.*; -import org.springframework.batch.item.*; -import org.springframework.batch.core.configuration.annotation.*; -import org.springframework.context.annotation.*; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.support.SimpleJobLauncher; +import org.springframework.batch.core.launch.support.TaskExecutorJobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.ItemStreamWriter; import org.springframework.batch.item.support.CompositeItemProcessor; import org.springframework.batch.item.support.CompositeItemWriter; import org.springframework.batch.support.transaction.ResourcelessTransactionManager; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; import org.springframework.core.io.Resource; import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.jdbc.datasource.init.DataSourceInitializer; @@ -62,7 +69,6 @@ */ @Configuration -@EnableBatchProcessing @ComponentScan(basePackages="org.mskcc.cmo.ks.redcap.source.internal") @PropertySource("classpath:application.properties") public class BatchConfiguration { @@ -75,97 +81,112 @@ public class BatchConfiguration { @Value("${chunk}") private Integer chunkInterval; - @Autowired - public JobBuilderFactory jobBuilderFactory; - - @Autowired - public StepBuilderFactory stepBuilderFactory; - - // Will keep calling clinicalDataStep or timelineDataStep based on the exit status from the clinicalDataStepListener - @Bean - public Job redcapExportJob() { - return jobBuilderFactory.get(REDCAP_EXPORT_JOB) - .start(exportClinicalDataStep()) - .next(exportTimelineDataStep()) + @Bean(name = "redcapExportJob") + public Job redcapExportJob(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("exportClinicalDataStep") Step exportClinicalDataStep, + @Qualifier("exportTimelineDataStep") Step exportTimelineDataStep) { + return new JobBuilder(REDCAP_EXPORT_JOB, jobRepository) + .preventRestart() + .start(exportClinicalDataStep) + .next(exportTimelineDataStep) .build(); } // Will keep calling clinicalDataStep or timelineDataStep based on the exit status from the clinicalDataStepListener - @Bean - public Job redcapRawExportJob() { - return jobBuilderFactory.get(REDCAP_RAW_EXPORT_JOB) - .start(exportRawClinicalDataStep()) + @Bean(name = "redcapRawExportJob") + public Job redcapRawExportJob(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("exportRawClinicalDataStep") Step exportRawClinicalDataStep, + @Qualifier("exportRawTimelineDataStep") Step exportRawTimelineDataStep) { + return new JobBuilder(REDCAP_RAW_EXPORT_JOB, jobRepository) + .preventRestart() + .start(exportRawClinicalDataStep) .on("CLINICAL") - .to(exportRawClinicalDataStep()) + .to(exportRawClinicalDataStep) .on("COMPLETED").end() .on("TIMELINE") - .to(exportRawTimelineDataStep()) + .to(exportRawTimelineDataStep) .on("TIMELINE") - .to(exportRawTimelineDataStep()) + .to(exportRawTimelineDataStep) .on("COMPLETED").end() .build() .build(); } - @Bean - public Job redcapImportJob() { - return jobBuilderFactory.get(REDCAP_IMPORT_JOB) - .start(importRedcapProjectDataStep()) + @Bean(name = "redcapImportJob") + public Job redcapImportJob(@Qualifier("redcapJobRepository") JobRepository jobRepository, @Qualifier("importRedcapProjectDataStep") Step importRedcapProjectDataStep) { + return new JobBuilder(REDCAP_IMPORT_JOB, jobRepository) + .preventRestart() + .start(importRedcapProjectDataStep) .build(); } @Bean - public Step exportClinicalDataStep() { - return stepBuilderFactory.get("exportClinicalDataStep") + protected Step exportClinicalDataStep(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("redcapTransactionManager") PlatformTransactionManager transactionManager, + @Qualifier("clinicalDataProcessor") CompositeItemProcessor, ClinicalDataComposite> processor, + @Qualifier("clinicalDataWriter") CompositeItemWriter writer) { + return new StepBuilder("exportClinicalDataStep", jobRepository) .listener(exportClinicalDataStepListener()) - ., ClinicalDataComposite> chunk(chunkInterval) + ., ClinicalDataComposite> chunk(chunkInterval, transactionManager) .reader(clinicalDataReader()) - .processor(clinicalDataProcessor()) - .writer(clinicalDataWriter()) + .processor(processor) + .writer(writer) .build(); } @Bean - public Step exportRawClinicalDataStep() { - return stepBuilderFactory.get("exportRawClinicalDataStep") + protected Step exportRawClinicalDataStep(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("redcapTransactionManager") PlatformTransactionManager transactionManager, + @Qualifier("rawClinicalDataProcessor") ItemProcessor, String> processor, + @Qualifier("rawClinicalDataWriter") ItemStreamWriter writer) { + return new StepBuilder("exportRawClinicalDataStep", jobRepository) .listener(exportRawClinicalDataStepListener()) - ., String> chunk(chunkInterval) + ., String> chunk(chunkInterval, transactionManager) .reader(clinicalDataReader()) - .processor(rawClinicalDataProcessor()) - .writer(rawClinicalDataWriter()) + .processor(processor) + .writer(writer) .build(); } @Bean - public Step exportTimelineDataStep() { - return stepBuilderFactory.get("exportTimelineDataStep") + protected Step exportTimelineDataStep(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("redcapTransactionManager") PlatformTransactionManager transactionManager, + @Qualifier("timelineProcessor") TimelineProcessor processor, + @Qualifier("timelineWriter") TimelineWriter writer) { + return new StepBuilder("exportTimelineDataStep", jobRepository) .listener(exportTimelineDataStepListener()) - ., String> chunk(chunkInterval) + ., String> chunk(chunkInterval, transactionManager) .reader(timelineReader()) - .processor(timelineProcessor()) - .writer(timelineWriter()) + .processor(processor) + .writer(writer) .build(); } @Bean - public Step exportRawTimelineDataStep() { - return stepBuilderFactory.get("exportRawTimelineDataStep") + protected Step exportRawTimelineDataStep(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("redcapTransactionManager") PlatformTransactionManager transactionManager, + @Qualifier("timelineProcessor") TimelineProcessor processor, + @Qualifier("timelineWriter") TimelineWriter writer) { + return new StepBuilder("exportRawTimelineDataStep", jobRepository) .listener(exportRawTimelineDataStepListener()) - ., String> chunk(chunkInterval) + ., String> chunk(chunkInterval, transactionManager) .reader(timelineReader()) - .processor(timelineProcessor()) - .writer(timelineWriter()) + .processor(processor) + .writer(writer) .build(); } + @Bean - public Step importRedcapProjectDataStep() { - return stepBuilderFactory.get("importRedcapProjectDataStep") - .tasklet(importRedcapProjectDataTasklet()) + protected Step importRedcapProjectDataStep(@Qualifier("redcapJobRepository") JobRepository jobRepository, + @Qualifier("redcapTransactionManager") PlatformTransactionManager transactionManager, + @Qualifier("importRedcapProjectData") Tasklet tasklet) { + return new StepBuilder("importRedcapProjectDataStep", jobRepository) + .tasklet(tasklet, transactionManager) .build(); } - @Bean + @Bean(name = "importRedcapProjectData") @StepScope public Tasklet importRedcapProjectDataTasklet() { return new ImportRedcapProjectDataTasklet(); @@ -183,9 +204,8 @@ public ItemStreamReader> clinicalDataReader() { // which contains the data necessary for the next processor/writer and the result of the processors. // The writers pull out the data they need from the composite result. - @Bean + @Bean(name = "clinicalDataProcessor") @StepScope - @SuppressWarnings("unchecked") public CompositeItemProcessor, ClinicalDataComposite> clinicalDataProcessor() { CompositeItemProcessor, ClinicalDataComposite> processor = new CompositeItemProcessor<>(); //TODO combine clinical data processors into a single class in order to avoid the need for a mixed type list @@ -196,15 +216,14 @@ public CompositeItemProcessor, ClinicalDataComposite> clinic return processor; } - @Bean + @Bean(name = "rawClinicalDataProcessor") @StepScope public RawClinicalDataProcessor rawClinicalDataProcessor() { return new RawClinicalDataProcessor(); } - @Bean + @Bean(name = "clinicalDataWriter") @StepScope - @SuppressWarnings("unchecked") public CompositeItemWriter clinicalDataWriter() { CompositeItemWriter writer = new CompositeItemWriter<>(); //TODO combine clinical data writers into a single class in order to avoid the need for a mixed type list @@ -215,7 +234,7 @@ public CompositeItemWriter clinicalDataWriter() { return writer; } - @Bean + @Bean(name = "rawClinicalDataWriter") @StepScope public ItemStreamWriter rawClinicalDataWriter() { return new RawClinicalDataWriter(); @@ -295,24 +314,6 @@ public RawTimelineDataStepListener exportRawTimelineDataStepListener() { @Value("org/springframework/batch/core/schema-sqlite.sql") private Resource dataRepositorySchema; - /** - * Spring Batch datasource. - * @return DataSource - */ - @Bean - public DataSource dataSource() { - DriverManagerDataSource dataSource = new DriverManagerDataSource(); - dataSource.setDriverClassName("org.sqlite.JDBC"); - dataSource.setUrl("jdbc:sqlite:repository.sqlite"); - return dataSource; - } - - /** - * Spring Batch datasource initializer. - * @param dataSource - * @return DataSourceInitializer - * @throws MalformedURLException - */ @Bean public DataSourceInitializer dataSourceInitializer(DataSource dataSource) throws MalformedURLException { ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator(); @@ -326,36 +327,25 @@ public DataSourceInitializer dataSourceInitializer(DataSource dataSource) throws return initializer; } - /** - * Spring Batch job repository. - * @return JobRepository - * @throws Exception - */ - private JobRepository getJobRepository() throws Exception { - JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean(); - factory.setDataSource(dataSource()); - factory.setTransactionManager(getTransactionManager()); - factory.afterPropertiesSet(); - return (JobRepository) factory.getObject(); + @Bean + public DataSource dataSource() { + final DriverManagerDataSource dataSource = new DriverManagerDataSource(); + dataSource.setDriverClassName("org.sqlite.JDBC"); + dataSource.setUrl("jdbc:sqlite:repository.sqlite"); + return dataSource; } - /** - * Spring Batch transaction manager. - * @return PlatformTransactionManager - */ - private PlatformTransactionManager getTransactionManager() { + @Bean(name = "redcapTransactionManager") + public PlatformTransactionManager getTransactionManager() { return new ResourcelessTransactionManager(); } - - /** - * Spring Batch job launcher. - * @return JobLauncher - * @throws Exception - */ - public JobLauncher getJobLauncher() throws Exception { - SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); - jobLauncher.setJobRepository(getJobRepository()); - jobLauncher.afterPropertiesSet(); - return jobLauncher; + + @Bean(name = "redcapJobRepository") + public JobRepository getJobRepository() throws Exception { + JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean(); + factory.setDataSource(dataSource()); + factory.setTransactionManager(getTransactionManager()); + factory.afterPropertiesSet(); + return factory.getObject(); } } diff --git a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalPatientDataWriter.java b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalPatientDataWriter.java index 814caf19a..229735a08 100644 --- a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalPatientDataWriter.java +++ b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalPatientDataWriter.java @@ -35,6 +35,7 @@ import java.io.*; import java.util.*; import org.apache.log4j.Logger; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.file.FlatFileHeaderCallback; import org.springframework.batch.item.file.FlatFileItemWriter; @@ -101,9 +102,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { if (writeClinicalPatient) { - List writeList = new ArrayList<>(); + Chunk writeList = new Chunk<>(); for (ClinicalDataComposite composite : items) { if (!writtenPatientSet.add(composite.getData().get("PATIENT_ID"))) { continue; diff --git a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalSampleDataWriter.java b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalSampleDataWriter.java index af4c92209..c11d94300 100644 --- a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalSampleDataWriter.java +++ b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/ClinicalSampleDataWriter.java @@ -35,6 +35,7 @@ import java.io.*; import java.util.*; import org.mskcc.cmo.ks.redcap.source.ClinicalDataSource; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.file.FlatFileHeaderCallback; import org.springframework.batch.item.file.FlatFileItemWriter; @@ -103,9 +104,9 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { if (writeClinicalSample) { - List writeList = new ArrayList<>(); + Chunk writeList = new Chunk<>(); for (ClinicalDataComposite composite : items) { writeList.add(composite.getSampleResult()); } diff --git a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/RawClinicalDataWriter.java b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/RawClinicalDataWriter.java index 1b3f08de9..d989f5ae5 100644 --- a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/RawClinicalDataWriter.java +++ b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/RawClinicalDataWriter.java @@ -35,6 +35,7 @@ import java.io.*; import java.util.*; import org.mskcc.cmo.ks.redcap.source.ClinicalDataSource; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -94,7 +95,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { if (writeRawClinicalData) { flatFileItemWriter.write(items); } diff --git a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/TimelineWriter.java b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/TimelineWriter.java index 95f1f7f3f..82b94d8a4 100644 --- a/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/TimelineWriter.java +++ b/redcap/redcap_pipeline/src/main/java/org/mskcc/cmo/ks/redcap/pipeline/TimelineWriter.java @@ -41,6 +41,7 @@ import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.file.FlatFileHeaderCallback; import org.springframework.batch.item.file.FlatFileItemWriter; import org.springframework.batch.item.file.transform.PassThroughLineAggregator; @@ -109,7 +110,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { if (writeTimelineData) { flatFileItemWriter.write(items); } diff --git a/redcap/redcap_source/pom.xml b/redcap/redcap_source/pom.xml index c696f9797..2179f8adb 100644 --- a/redcap/redcap_source/pom.xml +++ b/redcap/redcap_source/pom.xml @@ -3,11 +3,11 @@ MSKCC CMO Redcap Data Source Redcap Source redcap_source - 0.1.0 + 1.0.0 org.mskcc.cmo.ks.redcap.source org.mskcc.cmo.ks.redcap redcap - 0.1.0 + 1.0.0 diff --git a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/OverriddenCancerStudy.java b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/OverriddenCancerStudy.java index 69a359c7e..d65895bee 100644 --- a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/OverriddenCancerStudy.java +++ b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/OverriddenCancerStudy.java @@ -34,7 +34,7 @@ import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; +import jakarta.annotation.Generated; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapAttributeMetadata.java b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapAttributeMetadata.java index 171102f17..a8bcdce21 100644 --- a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapAttributeMetadata.java +++ b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapAttributeMetadata.java @@ -34,7 +34,7 @@ import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; +import jakarta.annotation.Generated; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapProjectAttribute.java b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapProjectAttribute.java index 0f5191cef..75019a100 100644 --- a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapProjectAttribute.java +++ b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapProjectAttribute.java @@ -34,7 +34,7 @@ import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; +import jakarta.annotation.Generated; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapToken.java b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapToken.java index a5cf03f5b..c88cb219a 100644 --- a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapToken.java +++ b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/models/RedcapToken.java @@ -34,7 +34,7 @@ import java.util.HashMap; import java.util.Map; -import javax.annotation.Generated; +import jakarta.annotation.Generated; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/source/internal/RedcapSessionManager.java b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/source/internal/RedcapSessionManager.java index f5244756c..f8c35a609 100644 --- a/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/source/internal/RedcapSessionManager.java +++ b/redcap/redcap_source/src/main/java/org/mskcc/cmo/ks/redcap/source/internal/RedcapSessionManager.java @@ -264,7 +264,7 @@ public void deleteRedcapProjectData(String token, Set recordNames) { } HttpEntity> requestEntity = getRequestEntity(uriVariables); ResponseEntity responseEntity = restTemplate.exchange(getRedcapApiURI(), HttpMethod.POST, requestEntity, String.class); - HttpStatus responseStatus = responseEntity.getStatusCode(); + HttpStatusCode responseStatus = responseEntity.getStatusCode(); if (!responseStatus.is2xxSuccessful()) { String errorMessage = "RedCap delete record API call failed. HTTP status code = " + Integer.toString(responseEntity.getStatusCode().value()); log.warn(errorMessage); @@ -301,7 +301,7 @@ public void importClinicalData(String token, String dataForImport) { importRecordUriVariables.add("data", dataForImport); HttpEntity> importRecordRequestEntity = getRequestEntity(importRecordUriVariables); ResponseEntity importRecordResponseEntity = restTemplate.exchange(getRedcapApiURI(), HttpMethod.POST, importRecordRequestEntity, String.class); - HttpStatus responseStatus = importRecordResponseEntity.getStatusCode(); + HttpStatusCode responseStatus = importRecordResponseEntity.getStatusCode(); if (!responseStatus.is2xxSuccessful()) { String message = "RedCap import record API call failed. HTTP status code = " + Integer.toString(importRecordResponseEntity.getStatusCode().value()); log.warn(message);