add transformer functions and mappers

.github/workflows/python-app.yml

@@ -48,7 +48,7 @@
        working-directory: ./tests
        run: |
          source ../.venv/bin/activate
-         pytest
+         pytest --db-url=${{ secrets.DB_URL }}
      - name: flake8 linter
        run:  |
          source ./.venv/bin/activate

README.md

@@ -26,4 +26,7 @@ pg_restore -d isb_sesar ./sesardb-schemaonly.dump
 ```
 
 ## Run the main script
-`python main.py -d "postgresql+psycopg2://isb_writer:password@localhost/isb_sesar"`
+`python main.py -d "postgresql+psycopg2://isb_writer:password@localhost/isb_sesar"`
+
+## Run the test script
+`pytest --db-url="postgresql+psycopg2://username:password@DB_HOST/DB_NAME"`

conftest.py

@@ -0,0 +1,4 @@
+def pytest_addoption(parser):
+    parser.addoption(
+        '--db-url', action='store'
+    )

examples/EOI00002H.json

@@ -50,10 +50,6 @@
                 "name": "Evans_Leigh",
                 "role": "collector"
             },
-            {
-                "name": "Andra Bobbitt",
-                "role": "sample owner"
-            },
             {
                 "name": "TN300",
                 "role": "sponsor"

examples/IEDUT103B.json

@@ -55,10 +55,6 @@
                 "name": "Andrea Dutton",
                 "role": "collector"
             },
-            {
-                "name": "Andrea Dutton",
-                "role": "sample owner"
-            },
             {
                 "name": "UF Jamaica 2015 Fieldwork",
                 "role": "sponsor"

examples/IEEJR000M.json

@@ -53,10 +53,6 @@
             {
                 "name": "Jade Star Lackey",
                 "role": "collector"
-            },
-            {
-                "name": "Evan Ramos",
-                "role": "sample owner"
             }
         ],
         "resultTime": "2021-01-19",

examples/IEJEN0040.json

@@ -49,10 +49,6 @@
             {
                 "name": "Miriam Jones",
                 "role": "collector"
-            },
-            {
-                "name": "Jonathan Nichols",
-                "role": "sample owner"
             }
         ],
         "resultTime": "2017-09-01",

examples/IERVTL1I7.json

@@ -10,8 +10,8 @@
         "label": "Not Provided",
         "responsibility": [
             {
-                "contact_information": "[email protected]",
-                "name": "Watershed Function SFA Data Team",
+                "contact_information": "[email protected]",
+                "name": "SLAC SFA",
                 "role": "sample owner"
             },
             {
@@ -50,10 +50,6 @@
                 "name": "Zach Perzan",
                 "role": "collector"
             },
-            {
-                "name": "SLAC SFA",
-                "role": "sample owner"
-            },
             {
                 "name": "SLAC-SFA",
                 "role": "sponsor"

examples/ODP02Q1IZ.json

@@ -50,10 +50,6 @@
         "hasFeatureOfInterest": "Not Provided",
         "label": "ODP Leg 178",
         "responsibility": [
-            {
-                "name": "Integrated Ocean Drilling Program (TAMU)",
-                "role": "sample owner"
-            },
             {
                 "name": "ODP Leg 178",
                 "role": "sponsor"

isamples_sesar/SesarTransformer.py → isamples_sesar/sesar_transformer.py

@@ -3,7 +3,7 @@
 import logging
 from .sample import Sample
 
-from .Mapper import (
+from .mapper import (
     AbstractCategoryMapper,
     StringPairedCategoryMapper,
     StringOrderedCategoryMapper,
@@ -112,9 +112,7 @@ def has_specimen_categories(self) -> typing.List[str]:
         return SpecimenCategoryMetaMapper.categories(sample_type)
 
     def id_string(self) -> str:
-        return "https://data.isamples.org/digitalsample/{0}/{1}".format(
-            'igsn', self.sample.igsn
-        )
+        return f"https://data.isamples.org/digitalsample/igsn/{self.sample.igsn}"
 
     def _material_type(self) -> str:
         if self.sample.classification and self.sample.top_level_classification:
@@ -158,6 +156,7 @@ def informal_classification(self) -> typing.List[str]:
         return [Transformer.NOT_PROVIDED]
 
     def keywords(self) -> typing.List:
+        # TODO: add more keywords
         keyword_arr = []
         sample_type = self.sample.sample_type
         if sample_type:
@@ -259,17 +258,17 @@ def produced_by_responsibilities(self) -> list[dict]:
             }
             responsibilities.append(collector)
 
-        if self.sample.orig_owner:
-            if self.sample.orig_owner.fname.lower() == 'curator':
-                sample_owner_name = self.sample.orig_owner.lname
-            else:
-                sample_owner_name = f"{self.sample.orig_owner.fname} {self.sample.orig_owner.lname}"
-            if sample_owner_name:
-                sample_owner = {
-                    "role": "sample owner",
-                    "name": sample_owner_name
-                }
-                responsibilities.append(sample_owner)
+        # if self.sample.orig_owner:
+        #     if self.sample.orig_owner.fname.lower() == 'curator':
+        #         sample_owner_name = self.sample.orig_owner.lname
+        #     else:
+        #         sample_owner_name = f"{self.sample.orig_owner.fname} {self.sample.orig_owner.lname}"
+        #     if sample_owner_name:
+        #         sample_owner = {
+        #             "role": "sample owner",
+        #             "name": sample_owner_name
+        #         }
+        #         responsibilities.append(sample_owner)
 
         if self.sample.cruise_field_prgrm:
             cruise_field_prgrm = {
@@ -601,7 +600,7 @@ class ContextCategoryMetaMapper(AbstractCategoryMetaMapper):
         "Liquid>aqueous", "Vent", "Subsurface fluid reservoir"
     )
     _floodplainAquiferMapper = StringPairedCategoryMapper(
-        "Liquid>aqueous", "floodplain\, aquifer", "Subsurface fluid reservoir"  # noqa: W605
+        "Liquid>aqueous", "floodplain, aquifer", "Subsurface fluid reservoir"  # noqa: W605
     )
     _creekBankMapper = StringPairedCategoryMapper(
         "Sedimentary>GlacialAndOrPaleosol>Rock",

main.py

@@ -1,6 +1,6 @@
 import click
 from isamples_sesar.sqlmodel_database import SQLModelDAO, get_sample_with_igsn
-from isamples_sesar.SesarTransformer import Transformer
+from isamples_sesar.sesar_transformer import Transformer
 import json
 
 

tests/test_isamples_metadata.py

@@ -0,0 +1,206 @@
+import pytest
+import json
+from sqlmodel import Session, create_engine
+
+from isamples_sesar.sesar_transformer import Transformer
+from isamples_sesar.sqlmodel_database import (
+    get_sample_with_igsn
+)
+
+
+@pytest.fixture
+def db_url(request):
+    return request.config.getoption("--db-url")
+
+
+@pytest.fixture(name="session")
+def session_fixture(db_url):
+    engine = create_engine(
+        db_url,
+        echo=False
+    )
+    with Session(engine) as session:
+        yield session
+
+
+@pytest.mark.parametrize("igsn", ["10.58052/EOI00002H",
+                                  "10.58052/IEDUT103B",
+                                  "10.58052/IEEJR000M",
+                                  "10.58052/IEJEN0040",
+                                  "10.58052/IERVTL1I7",
+                                  "10.60471/ODP02Q1IZ"])
+def test_example(session: Session, igsn):
+    # get sample and transform it
+    sample = get_sample_with_igsn(session, igsn)
+    assert sample is not None
+    transformed_test_data = Transformer(sample).transform()
+
+    igsn_suffix = igsn.split("/")[1]
+    json_file = open("examples/"+igsn_suffix+".json")
+    expected_data = json.load(json_file)
+    json_file.close()
+
+    check_id(transformed_test_data, expected_data)
+    check_label(transformed_test_data, expected_data)
+    check_sample_identifier(transformed_test_data, expected_data)
+    check_description(transformed_test_data, expected_data)
+    check_context_category(transformed_test_data, expected_data)
+    check_material_category(transformed_test_data, expected_data)
+    check_specimen_category(transformed_test_data, expected_data)
+    check_informal_classification(transformed_test_data, expected_data)
+    check_keywords(transformed_test_data, expected_data)
+    check_produced_by_id(transformed_test_data, expected_data)
+    check_produced_by_label(transformed_test_data, expected_data)
+    check_produced_by_description(transformed_test_data, expected_data)
+    check_produced_by_feature(transformed_test_data, expected_data)
+    check_produced_by_responsibility(transformed_test_data, expected_data)
+    check_produced_by_time(transformed_test_data, expected_data)
+    check_sampling_site_description(transformed_test_data, expected_data)
+    check_sampling_site_label(transformed_test_data, expected_data)
+    check_sampling_site_place_name(transformed_test_data, expected_data)
+    check_sampling_site_elevation(transformed_test_data, expected_data)
+    check_sampling_site_latitude(transformed_test_data, expected_data)
+    check_sampling_site_longitude(transformed_test_data, expected_data)
+    check_registrant(transformed_test_data, expected_data)
+    check_sampling_purpose(transformed_test_data, expected_data)
+    check_curation_label(transformed_test_data, expected_data)
+    check_curation_description(transformed_test_data, expected_data)
+    check_curation_access_constraints(transformed_test_data, expected_data)
+    check_curation_location(transformed_test_data, expected_data)
+    check_curation_responsibility(transformed_test_data, expected_data)
+    check_related_resource(transformed_test_data, expected_data)
+    check_authorized_by(transformed_test_data, expected_data)
+    check_complies_with(transformed_test_data, expected_data)
+
+
+def check_id(test_data, expected_data):
+    assert test_data["@id"] == expected_data["@id"]
+
+
+def check_label(test_data, expected_data):
+    assert test_data["label"] == expected_data["label"]
+
+
+def check_sample_identifier(test_data, expected_data):
+    assert test_data["sampleidentifier"] == expected_data["sampleidentifier"]
+
+
+def check_description(test_data, expected_data):
+    assert test_data["description"] == expected_data["description"]
+
+
+def check_context_category(test_data, expected_data):
+    assert test_data["hasContextCategory"] == expected_data["hasContextCategory"]
+    # assert test_data["hasContextCategoryConfidence"] == expected_data["hasContextCategoryConfidence"]
+
+
+def check_material_category(test_data, expected_data):
+    assert test_data["hasMaterialCategory"] == expected_data["hasMaterialCategory"]
+    # assert test_data["hasMaterialCategoryConfidence"] == expected_data["hasMaterialCategoryConfidence"]
+
+
+def check_specimen_category(test_data, expected_data):
+    assert test_data["hasSpecimenCategory"] == expected_data["hasSpecimenCategory"]
+    # assert test_data["hasSpecimenCategoryConfidence"] == expected_data["hasSpecimenCategoryConfidence"]
+
+
+def check_informal_classification(test_data, expected_data):
+    assert test_data["informalClassification"] == expected_data["informalClassification"]
+
+
+def check_keywords(test_data, expected_data):
+    assert test_data["keywords"] == expected_data["keywords"]
+
+
+def check_produced_by_id(test_data, expected_data):
+    assert test_data["producedBy"]["@id"] == expected_data["producedBy"]["@id"]
+
+
+def check_produced_by_label(test_data, expected_data):
+    assert test_data["producedBy"]["label"] == expected_data["producedBy"]["label"]
+
+
+def check_produced_by_description(test_data, expected_data):
+    assert test_data["producedBy"]["description"] == expected_data["producedBy"]["description"]
+
+
+def check_produced_by_feature(test_data, expected_data):
+    assert test_data["producedBy"]["hasFeatureOfInterest"] == expected_data["producedBy"]["hasFeatureOfInterest"]
+
+
+def check_produced_by_responsibility(test_data, expected_data):
+    assert test_data["producedBy"]["responsibility"] == expected_data["producedBy"]["responsibility"]
+
+
+def check_produced_by_time(test_data, expected_data):
+    assert test_data["producedBy"]["resultTime"] == expected_data["producedBy"]["resultTime"]
+
+
+def check_sampling_site_description(test_data, expected_data):
+    assert test_data["producedBy"]["samplingSite"]["description"] == \
+        expected_data["producedBy"]["samplingSite"]["description"]
+
+
+def check_sampling_site_label(test_data, expected_data):
+    assert test_data["producedBy"]["samplingSite"]["label"] == \
+        expected_data["producedBy"]["samplingSite"]["label"]
+
+
+def check_sampling_site_place_name(test_data, expected_data):
+    assert test_data["producedBy"]["samplingSite"]["placeName"] == \
+        expected_data["producedBy"]["samplingSite"]["placeName"]
+
+
+def check_sampling_site_elevation(test_data, expected_data):
+    assert test_data["producedBy"]["samplingSite"]["location"]["elevation"] == \
+        expected_data["producedBy"]["samplingSite"]["location"]["elevation"]
+
+
+def check_sampling_site_latitude(test_data, expected_data):
+    assert test_data["producedBy"]["samplingSite"]["location"]["latitude"] == \
+        expected_data["producedBy"]["samplingSite"]["location"]["latitude"]
+
+
+def check_sampling_site_longitude(test_data, expected_data):
+    assert test_data["producedBy"]["samplingSite"]["location"]["longitude"] == \
+        expected_data["producedBy"]["samplingSite"]["location"]["longitude"]
+
+
+def check_registrant(test_data, expected_data):
+    assert test_data["registrant"] == expected_data["registrant"]
+
+
+def check_sampling_purpose(test_data, expected_data):
+    assert test_data["samplingPurpose"] == expected_data["samplingPurpose"]
+
+
+def check_curation_label(test_data, expected_data):
+    assert test_data["curation"]["label"] == expected_data["curation"]["label"]
+
+
+def check_curation_description(test_data, expected_data):
+    assert test_data["curation"]["description"] == expected_data["curation"]["description"]
+
+
+def check_curation_access_constraints(test_data, expected_data):
+    assert test_data["curation"]["accessConstraints"] == expected_data["curation"]["accessConstraints"]
+
+
+def check_curation_location(test_data, expected_data):
+    assert test_data["curation"]["curationLocation"] == expected_data["curation"]["curationLocation"]
+
+
+def check_curation_responsibility(test_data, expected_data):
+    assert test_data["curation"]["responsibility"] == expected_data["curation"]["responsibility"]
+
+
+def check_related_resource(test_data, expected_data):
+    assert test_data["relatedResource"] == expected_data["relatedResource"]
+
+
+def check_authorized_by(test_data, expected_data):
+    assert test_data["authorizedBy"] == expected_data["authorizedBy"]
+
+
+def check_complies_with(test_data, expected_data):
+    assert test_data["compliesWith"] == expected_data["compliesWith"]