diff --git a/clinvar_this/batches.py b/clinvar_this/batches.py index 81113f4..4ccbcf6 100644 --- a/clinvar_this/batches.py +++ b/clinvar_this/batches.py @@ -273,9 +273,9 @@ def retrieve(config: config.Config, name: str, *, use_testing: bool = False): submission_path = get_share_dir() / config.profile / name submission_response_paths = list(sorted(submission_path.glob("submission-response.*.json"))) - if not submission_path.exists(): + if not submission_path.exists(): # pragma: no cover raise exceptions.ClinvarThisException(f"Submission does not exist at {submission_path}") - elif not submission_response_paths: + elif not submission_response_paths: # pragma: no cover raise exceptions.ClinvarThisException( f"Submission not submitted? No submission response at {submission_path}" ) @@ -311,6 +311,6 @@ def retrieve(config: config.Config, name: str, *, use_testing: bool = False): logger.info("Will now update local information from response...") _retrieve_store_response(config, name, status_result) logger.info("... done updating local information from response") - else: + else: # pragma: no cover logger.error("Status is %s and clinvar-this does not know how to handle this yet!") raise exceptions.ClinvarThisException(f"Unknown status {status_str}") diff --git a/tests/clinvar_this/data/batches/small_variant.retrieve-response-error.json b/tests/clinvar_this/data/batches/small_variant.retrieve-response-error.json new file mode 100644 index 0000000..640a92f --- /dev/null +++ b/tests/clinvar_this/data/batches/small_variant.retrieve-response-error.json @@ -0,0 +1,173 @@ +{ + "status": { + "actions": [ + { + "id": "SUB000fake-1", + "responses": [ + { + "status": "error", + "files": [ + { + "url": "https://submit.ncbi.nlm.nih.gov/api/2.0/files/fake/subfake-summary-report.json/?format=attachment" + } + ], + "message": { + "error_code": "1", + "severity": "error", + "text": "Your ClinVar submission processing status is \"Partial success\". Please find the details in the file referenced by actions[0].responses[0].files[0].url." + }, + "objects": [] + } + ], + "status": "error", + "target_db": "clinvar", + "updated": "2022-12-02T10:18:04+01:00" + } + ] + }, + "summaries": { + "https://submit.ncbi.nlm.nih.gov/api/2.0/files/fake/subfake-summary-report.json/?format=attachment": { + "batch_processing_status": "Partial success", + "batch_release_status": "Not released", + "submission_date": "2022-12-02", + "submission_name": "SUB000fake", + "total_count": 2, + "total_errors": 1, + "total_public": 0, + "total_success": 1, + "deletions": null, + "submissions": [ + { + "identifiers": { + "clinvar_local_key": "fake", + "clinvar_accession": null, + "local_id": "fake", + "local_key": null + }, + "processing_status": "Error", + "clinvar_accession_version": null, + "errors": [ + { + "input": [ + { + "value": "Variation", + "field": "Variant.type" + }, + { + "value": null, + "field": "HGVS.accession" + }, + { + "value": null, + "field": "HGVS.change" + }, + { + "value": null, + "field": "HGVS.hgvs" + }, + { + "value": "GRCh37", + "field": "SequenceLocation.assembly" + }, + { + "value": "11", + "field": "SequenceLocation.chr" + }, + { + "value": "123", + "field": "SequenceLocation.start" + }, + { + "value": "1223", + "field": "SequenceLocation.stop" + }, + { + "value": "T", + "field": "SequenceLocation.reference_allele" + }, + { + "value": "A", + "field": "SequenceLocation.alternate_allele" + }, + { + "value": null, + "field": "SequenceLocation.outer_start" + }, + { + "value": null, + "field": "SequenceLocation.inner_start" + }, + { + "value": null, + "field": "SequenceLocation.inner_stop" + }, + { + "value": null, + "field": "SequenceLocation.outer_stop" + }, + { + "value": null, + "field": "SequenceLocation.variant_length" + }, + { + "value": null, + "field": "SequenceLocation.copy_number" + }, + { + "value": null, + "field": "SequenceLocation.reference_copy_number" + }, + { + "value": null, + "field": "SequenceLocation.breakpoint1" + }, + { + "value": null, + "field": "SequenceLocation.breakpoint2" + }, + { + "value": null, + "field": "Condition.name" + }, + { + "value": "OMIM", + "field": "ConditionID.type" + }, + { + "value": "12345", + "field": "ConditionID.value" + } + ], + "output": { + "errors": [ + { + "user_message": "This record is submitted as novel but it should be submitted as an update, including the SCV accession, because your organization previously submitted SCVfake for the same variant and condition." + } + ] + } + } + ], + "release_date": null, + "release_status": null + }, + { + "identifiers": { + "clinvar_local_key": "fake", + "clinvar_accession": "SCVfake", + "local_id": "fake", + "local_key": "fake" + }, + "processing_status": "Success", + "clinvar_accession_version": null, + "errors": null, + "release_date": null, + "release_status": null + } + ], + "total_delete_count": null, + "total_deleted": null, + "total_delete_errors": null, + "total_delete_success": null + } + } +} diff --git a/tests/clinvar_this/data/batches/small_variant.retrieve-response-processing.json b/tests/clinvar_this/data/batches/small_variant.retrieve-response-processing.json new file mode 100644 index 0000000..5574d00 --- /dev/null +++ b/tests/clinvar_this/data/batches/small_variant.retrieve-response-processing.json @@ -0,0 +1,25 @@ +{ + "status": { + "actions": [ + { + "id": "SUB000fake-1", + "responses": [ + { + "status": "processing", + "files": [], + "message": { + "error_code": null, + "severity": "info", + "text": "Your ClinVar submission processing status is \"In processing\"." + }, + "objects": [] + } + ], + "status": "processing", + "target_db": "clinvar", + "updated": "2022-11-18T16:25:05+01:00" + } + ] + }, + "summaries": {} +} diff --git a/tests/clinvar_this/data/batches/small_variant.retrieve-response-submitted.json b/tests/clinvar_this/data/batches/small_variant.retrieve-response-submitted.json new file mode 100644 index 0000000..adf02f8 --- /dev/null +++ b/tests/clinvar_this/data/batches/small_variant.retrieve-response-submitted.json @@ -0,0 +1,14 @@ +{ + "status": { + "actions": [ + { + "id": "SUB000fake-1", + "responses": [], + "status": "submitted", + "target_db": "clinvar", + "updated": "2022-11-18T16:24:19+01:00" + } + ] + }, + "summaries": {} +} diff --git a/tests/clinvar_this/data/batches/small_variant.retrieve-response-success.json b/tests/clinvar_this/data/batches/small_variant.retrieve-response-success.json new file mode 100644 index 0000000..c38b09c --- /dev/null +++ b/tests/clinvar_this/data/batches/small_variant.retrieve-response-success.json @@ -0,0 +1,60 @@ +{ + "status": { + "actions": [ + { + "id": "SUB000fake-1", + "responses": [ + { + "status": "processed", + "files": [ + { + "url": "https://submit.ncbi.nlm.nih.gov/api/2.0/files/fake/fake/?format=attachment" + } + ], + "message": { + "error_code": "0", + "severity": "info", + "text": "Your ClinVar submission processing status is \"Success\". Please find the details in the file referenced by actions[0].responses[0].files[0].url." + }, + "objects": [] + } + ], + "status": "processed", + "target_db": "clinvar", + "updated": "2022-11-18T12:52:11+01:00" + } + ] + }, + "summaries": { + "https://submit.ncbi.nlm.nih.gov/api/2.0/files/fake/fake/?format=attachment": { + "batch_processing_status": "Success", + "batch_release_status": "Not released", + "submission_date": "2022-11-18", + "submission_name": "SUB000fake", + "total_count": 8, + "total_errors": 0, + "total_public": 0, + "total_success": 8, + "deletions": null, + "submissions": [ + { + "identifiers": { + "clinvar_local_key": "clinvar-local-key", + "clinvar_accession": "SCVfake", + "local_id": "mock-uuid4-local-id", + "local_key": "mock-uuid4-local-key" + }, + "processing_status": "Success", + "clinvar_accession_version": null, + "errors": null, + "release_date": null, + "release_status": null + } + ], + "total_delete_count": null, + "total_deleted": null, + "total_delete_errors": null, + "total_delete_success": null + } + } +} diff --git a/tests/clinvar_this/test_batches.py b/tests/clinvar_this/test_batches.py index 9078119..31272c3 100644 --- a/tests/clinvar_this/test_batches.py +++ b/tests/clinvar_this/test_batches.py @@ -1,10 +1,15 @@ +import json import os import pathlib +from unittest.mock import MagicMock from freezegun import freeze_time import pytest import clinvar_api +from clinvar_api import models +from clinvar_api.client import RetrieveStatusResult +from clinvar_api.common import CONVERTER from clinvar_this import batches, exceptions from clinvar_this.io import tsv as io_tsv @@ -29,6 +34,30 @@ #: The `SMALL_VARIANT_UPDATE_TSV` after import / merge. SMALL_VARIANT_UPDATE_PAYLOAD_JSON = inputf.read() +with ( + pathlib.Path(__file__).parent / "data/batches/small_variant.retrieve-response-processing.json" +).open("rt") as inputf: + #: A static response on retrieving with "processing" status. + SMALL_VARIANT_RETRIEVE_RESPONSE_PROCESSING_JSON = inputf.read() + +with ( + pathlib.Path(__file__).parent / "data/batches/small_variant.retrieve-response-submitted.json" +).open("rt") as inputf: + #: A static response on retrieving with "submitted" status. + SMALL_VARIANT_RETRIEVE_RESPONSE_SUBMITTED_JSON = inputf.read() + +with ( + pathlib.Path(__file__).parent / "data/batches/small_variant.retrieve-response-success.json" +).open("rt") as inputf: + #: A static response on retrieving with "success" status. + SMALL_VARIANT_RETRIEVE_RESPONSE_SUCCESS_JSON = inputf.read() + +with ( + pathlib.Path(__file__).parent / "data/batches/small_variant.retrieve-response-error.json" +).open("rt") as inputf: + #: A static response on retrieving with "error" status. + SMALL_VARIANT_RETRIEVE_RESPONSE_ERROR_JSON = inputf.read() + SUBMISSION_SCHEMA_JSON_PATH = ( pathlib.Path(clinvar_api.__file__).parent / "schemas/submission_schema.json" ) @@ -133,19 +162,19 @@ def test_import_small_variant_tsv_update(fs, app_config): def test_import_deletion_tsv_new(fs): - pass + pass # TODO def test_import_deletion_tsv_update(fs): - pass + pass # TODO def test_import_structural_variant_tsv_new(fs): - pass + pass # TODO def test_import_structural_variant_tsv_update(fs): - pass + pass # TODO @pytest.mark.parametrize( @@ -194,7 +223,7 @@ def test_export_small_variant_tsv(fs, app_config, exists, force): def test_export_structural_variant_tsv(fs): - pass + pass # TODO @pytest.mark.parametrize( @@ -215,7 +244,7 @@ def test_submit(fs, app_config, use_testing, dry_run, monkeypatch): def mock_submit_data(_self, _payload): return {"id": "SUB000fake"} - monkeypatch.setattr(batches.client, "submit_data", mock_submit_data) + monkeypatch.setattr(batches.client.Client, "submit_data", mock_submit_data) batches.submit(config=app_config, name="the-batch", use_testing=use_testing, dry_run=dry_run) @@ -230,17 +259,129 @@ def mock_submit_data(_self, _payload): assert not os.path.exists(response_path) -def test_retrieve_state_submitted(fs): - pass +def test_retrieve_state_submitted(fs, app_config, monkeypatch): + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/payload.20120113000000.json" + ), + contents=SMALL_VARIANT_PAYLOAD_JSON, + ) + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/submission-response.20120114000000.json" + ), + contents='{"id": "SUB000fake"}', + ) + + response = json.loads(SMALL_VARIANT_RETRIEVE_RESPONSE_SUBMITTED_JSON) + + mock_retrieve_status = MagicMock() + mock_retrieve_status.return_value = RetrieveStatusResult( + status=CONVERTER.structure(response["status"], models.SubmissionStatus), summaries={} + ) + + monkeypatch.setattr(batches.client.Client, "retrieve_status", mock_retrieve_status) + + batches.retrieve(config=app_config, name="the-batch", use_testing=False) + + mock_retrieve_status.assert_called_once() + assert len(mock_retrieve_status.call_args.args) == 1 + assert len(mock_retrieve_status.call_args.kwargs) == 0 + assert mock_retrieve_status.call_args.args[0] == "SUB000fake" + + +def test_retrieve_state_processing(fs, app_config, monkeypatch): + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/payload.20120113000000.json" + ), + contents=SMALL_VARIANT_PAYLOAD_JSON, + ) + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/submission-response.20120114000000.json" + ), + contents='{"id": "SUB000fake"}', + ) + + response = json.loads(SMALL_VARIANT_RETRIEVE_RESPONSE_PROCESSING_JSON) + + mock_retrieve_status = MagicMock() + mock_retrieve_status.return_value = RetrieveStatusResult( + status=CONVERTER.structure(response["status"], models.SubmissionStatus), summaries={} + ) + + monkeypatch.setattr(batches.client.Client, "retrieve_status", mock_retrieve_status) + batches.retrieve(config=app_config, name="the-batch", use_testing=False) -def test_retrieve_state_processing(fs): - pass + mock_retrieve_status.assert_called_once() + assert len(mock_retrieve_status.call_args.args) == 1 + assert len(mock_retrieve_status.call_args.kwargs) == 0 + assert mock_retrieve_status.call_args.args[0] == "SUB000fake" -def test_retrieve_state_processed(fs): - pass +def test_retrieve_state_processed(fs, app_config, monkeypatch): + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/payload.20120113000000.json" + ), + contents=SMALL_VARIANT_PAYLOAD_JSON, + ) + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/submission-response.20120114000000.json" + ), + contents='{"id": "SUB000fake"}', + ) + + response = json.loads(SMALL_VARIANT_RETRIEVE_RESPONSE_SUCCESS_JSON) + + mock_retrieve_status = MagicMock() + mock_retrieve_status.return_value = RetrieveStatusResult( + status=CONVERTER.structure(response["status"], models.SubmissionStatus), + summaries={ + key: CONVERTER.structure(value, models.SummaryResponse) + for key, value in response["summaries"].items() + }, + ) + + monkeypatch.setattr(batches.client.Client, "retrieve_status", mock_retrieve_status) + + batches.retrieve(config=app_config, name="the-batch", use_testing=False) + + mock_retrieve_status.assert_called_once() + assert len(mock_retrieve_status.call_args.args) == 1 + assert len(mock_retrieve_status.call_args.kwargs) == 0 + assert mock_retrieve_status.call_args.args[0] == "SUB000fake" + + +def test_retrieve_state_error(fs, app_config, monkeypatch): + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/payload.20120113000000.json" + ), + contents=SMALL_VARIANT_PAYLOAD_JSON, + ) + fs.create_file( + os.path.expanduser( + "~/.local/share/clinvar-this/default/the-batch/submission-response.20120114000000.json" + ), + contents='{"id": "SUB000fake"}', + ) + + response = json.loads(SMALL_VARIANT_RETRIEVE_RESPONSE_ERROR_JSON) + + mock_retrieve_status = MagicMock() + mock_retrieve_status.return_value = RetrieveStatusResult( + status=CONVERTER.structure(response["status"], models.SubmissionStatus), summaries={} + ) + + monkeypatch.setattr(batches.client.Client, "retrieve_status", mock_retrieve_status) + batches.retrieve(config=app_config, name="the-batch", use_testing=False) -def test_retrieve_state_error(fs): - pass + mock_retrieve_status.assert_called_once() + assert len(mock_retrieve_status.call_args.args) == 1 + assert len(mock_retrieve_status.call_args.kwargs) == 0 + assert mock_retrieve_status.call_args.args[0] == "SUB000fake"