diff --git a/tests/extraction_methods/__init__.py b/tests/extraction_methods/__init__.py deleted file mode 100644 index 2f0b3e2..0000000 --- a/tests/extraction_methods/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "03 Dec 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" diff --git a/tests/extraction_methods/test_post_processors.py b/tests/extraction_methods/test_post_processors.py deleted file mode 100644 index 5f9c4ad..0000000 --- a/tests/extraction_methods/test_post_processors.py +++ /dev/null @@ -1,288 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "15 Jul 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" - -import pytest - -from stac_generator.plugins.postprocessors.bbox import BboxPostProcessor -from stac_generator.plugins.postprocessors.date_combinator import ( - DateCombinatorPostProcessor, -) -from stac_generator.plugins.postprocessors.facet_map import FacetMapPostProcessor -from stac_generator.plugins.postprocessors.facet_prefix import FacetPrefixPostProcessor -from stac_generator.plugins.postprocessors.iso_date import ISODatePostProcessor - - -@pytest.fixture -def fpath(): - return "a/b/c/d.txt" - - -@pytest.fixture -def body(): - return {"date": "2021-05-02"} - - -@pytest.fixture -def isodate_processor(): - return ISODatePostProcessor(date_keys=["date"]) - - -@pytest.fixture -def isodate_processor_with_format(): - return ISODatePostProcessor(date_keys=["date"], format="%Y%m") - - -@pytest.fixture -def facet_map_processor(): - return FacetMapPostProcessor(term_map={"date": "start_date"}) - - -@pytest.fixture -def bbox_processor(): - return BboxPostProcessor(coordinate_keys=["west", "south", "east", "north"]) - - -@pytest.fixture -def facet_prefix_processor(): - return FacetPrefixPostProcessor(prefix="CMIP6", terms=["date"]) - - -def test_isodate_processor(isodate_processor, fpath, body): - """Check isodate processor does what's expected""" - expected = body.copy() - expected["date"] = "2021-05-02T00:00:00" - - output = isodate_processor.run(fpath, body=body) - assert output == expected - - -def test_isodate_processor_bad_date(isodate_processor, fpath, caplog): - """Check isodate processor does what's expected.date - - Conditions: - - No format string - - Date string not parsable by dateutil - - Expected: - - Produce an error - - Delete the date key - """ - body = {"date": "202105"} - expected = {} - - output = isodate_processor.run(fpath, body=body) - assert output == expected - assert len(caplog.records) == 2 - assert caplog.records[0].levelname == "ERROR" - - -def test_isodate_processor_with_good_format(isodate_processor_with_format, fpath): - """Check isodate processor. - Conditions: - - Format string does match date string - - Date string not parsable by dateutil - - Expected: - - Successfully use strptime to parse the date - """ - - body = {"date": "202105"} - expected = body.copy() - expected["date"] = "2021-05-01T00:00:00" - - output = isodate_processor_with_format.run(fpath, body=body) - assert output == expected - - -def test_isodate_processor_with_bad_format( - isodate_processor_with_format, fpath, caplog -): - """Check isodate processor. - Conditions: - - Format string does not match date string - - Date string parsable by dateutil - - Expected: - - Produce warning - - Successfully Use dateutil to parse the date - """ - body = {"date": "20210501"} - expected = body.copy() - expected["date"] = "2021-05-01T00:00:00" - - output = isodate_processor_with_format.run(fpath, body=body) - assert output == expected - assert len(caplog.records) == 2 - assert caplog.records[0].levelname == "WARNING" - - -def test_isodate_processor_with_bad_format_bad_dateutil( - isodate_processor_with_format, fpath, caplog -): - """Check isodate processor. - Conditions: - - Format string does not match date string - - Date string ambiguous and nor parsable by dateutil - - Expected: - Should delete the date - """ - body = {"date": "2021010101"} - expected = {} - - output = isodate_processor_with_format.run(fpath, body=body) - assert output == expected - assert len(caplog.records) == 4 - - -def test_facet_map_processor(facet_map_processor, fpath, body): - """ - Check processor changes name of named facets - """ - expected = {"start_date": body["date"]} - - output = facet_map_processor.run(fpath, body=body) - assert output == expected - - -def test_bbox_processor(bbox_processor, fpath): - body = {"north": "42.0", "south": "38.0", "east": "-28.0", "west": "-37.0"} - - expected = { - "bbox": { - "type": "envelope", - "coordinates": [ - [ - float(body["west"]), - float(body["south"]), - ], - [ - float(body["east"]), - float(body["north"]), - ], - ], - }, - "north": "42.0", - "south": "38.0", - "east": "-28.0", - "west": "-37.0", - } - - output = bbox_processor.run(fpath, body=body) - assert output == expected - - -def test_date_combinator_no_year(fpath, caplog): - """ - Test that not providing a year results in an error logged - """ - processor = DateCombinatorPostProcessor() - - body = {"month": "02", "day": "01"} - - processor.run(fpath, body=body) - assert len(caplog.records) == 1 - assert caplog.records[0].levelname == "ERROR" - - -def test_date_combinator(fpath): - """ - Test can join and produce correct string - """ - processor = DateCombinatorPostProcessor() - - body = {"year": "1850", "month": "02", "day": "01"} - - expected = {} - expected["datetime"] = "1850-02-01T00:00:00" - - output = processor.run(fpath, body=body) - assert output == expected - - -def test_date_combinator_non_destructive(fpath): - """ - Test that data is preserved in non-destrictive mode - """ - processor = DateCombinatorPostProcessor(destructive=False) - - body = {"year": "1850", "month": "02", "day": "01"} - - expected = body.copy() - expected["datetime"] = "1850-02-01T00:00:00" - - output = processor.run(fpath, body=body) - assert output == expected - - -def test_date_combinator_format_string(fpath): - """ - Test can use format string - """ - processor = DateCombinatorPostProcessor(format="%Y-%m") - - body = { - "year": "1850", - "month": "02", - } - - expected = {} - expected["datetime"] = "1850-02-01T00:00:00" - - output = processor.run(fpath, body=body) - assert output == expected - - -def test_date_combinator_no_kwargs(fpath): - """Check that the processor can run with no kwargs.""" - processor = DateCombinatorPostProcessor() - - body = {"year": "1850", "month": "02", "day": "01"} - - expected = {} - expected["datetime"] = "1850-02-01T00:00:00" - - output = processor.run(fpath, body=body) - assert output == expected - - -def test_date_combinator_ym_no_format(fpath, caplog): - """ - Check that the processor logs an error - if only year and month are provided with - no format string. - - Expected: - Should keep the original data - Should produce an error log - """ - - processor = DateCombinatorPostProcessor() - - body = { - "year": "1850", - "month": "02", - } - - expected = body.copy() - - output = processor.run(fpath, body=body) - assert output == expected - assert len(caplog.records) == 1 - - -def test_facet_prefix_processor(facet_prefix_processor, fpath, body): - """ - Check processor adds prefix to named facets - """ - expected = {"CMIP6:date": body["date"]} - - output = facet_prefix_processor.run(fpath, body=body) - assert output == expected diff --git a/tests/extraction_methods/test_pre_processors.py b/tests/extraction_methods/test_pre_processors.py deleted file mode 100644 index e6bf6de..0000000 --- a/tests/extraction_methods/test_pre_processors.py +++ /dev/null @@ -1,53 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "15 Jul 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" - -import pytest - -from stac_generator.plugins.preprocessors.basename import BasenamePreProcessor -from stac_generator.plugins.preprocessors.ceda_observation import ( - CEDAObservationPreProcessor, -) - - -@pytest.fixture -def basename(): - """ - Create basename instance - :return: - """ - return BasenamePreProcessor() - - -def test_basename_posix(basename): - """ - Check that pre-processor extracts the filename from the POSIX path - :param basename: - :return: - """ - input_path = "/a/b/c/d.txt" - expected = "d.txt" - - uri, _ = basename.run(input_path) - assert uri == expected - - -@pytest.fixture -def ceda_observation(): - return CEDAObservationPreProcessor( - url_template="http://api.catalogue.ceda.ac.uk/api/v0/obs/get_info$uri" - ) - - -def test_ceda_observation(ceda_observation): - input_path = "/badc/faam/data/2005/b070-jan-06" - expected = "6f6d4b4fc7a042568cce7eccc6e9b6f2" - - _, kwargs = ceda_observation.run(input_path) - assert kwargs["uuid"] == expected diff --git a/tests/extraction_methods/test_processors.py b/tests/extraction_methods/test_processors.py deleted file mode 100644 index 70632d3..0000000 --- a/tests/extraction_methods/test_processors.py +++ /dev/null @@ -1,89 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "15 Jul 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" - -import pytest - -from stac_generator.plugins.extraction_methods.iso19115 import ISO19115Extract -from stac_generator.plugins.extraction_methods.regex import RegexExtract -from stac_generator.plugins.preprocessors.ceda_observation import ( - CEDAObservationPreProcessor, -) - - -@pytest.fixture -def regex_processor(): - return RegexExtract(regex=r"^(?:\w*[\s]){2}(?P\w*)") - - -@pytest.fixture -def iso19115_processor(): - return ISO19115Extract( - url_template="http://api.catalogue.ceda.ac.uk/export/xml/$uuid.xml", - extraction_keys=[{"name": "start_datetime", "key": ".//gml:beginPosition"}], - ) - - -@pytest.fixture -def ceda_observation(): - return CEDAObservationPreProcessor( - url_template="http://api.catalogue.ceda.ac.uk/api/v0/obs/get_info$uri" - ) - - -def test_regex_extract(regex_processor): - """Check can extract groups with regex""" - - input = "this contains interesting stuff" - expected = {"interesting": "interesting"} - - output = regex_processor.run(input) - assert output == expected - - -# TODO: Mock the call to catalogue server -def test_iso19115_extract(iso19115_processor): - """Check can extract data from the iso record""" - - input = "/irrelevant/filepath" - expected = {"start_datetime": "2005-01-06T09:57:05"} - - iso19115_processor.url_template = "http://api.catalogue.ceda.ac.uk/export/xml/6f6d4b4fc7a042568cce7eccc6e9b6f2.xml" - - output = iso19115_processor.run(input) - assert output == expected - - -# TODO: Mock the call to catalogue server -def test_iso19115_processor_with_preprocessor(iso19115_processor, ceda_observation): - """Check processor works with ceda obs pre-processor""" - - input = "/badc/faam/data/2005/b070-jan-06" - expected = {"start_datetime": "2005-01-06T09:57:05"} - - output = iso19115_processor.run(input, pre_processors=[ceda_observation]) - assert output == expected - - -# TODO: Mock the call to catalogue server -def test_iso19115_processor_with_preprocessor_null_output( - iso19115_processor, ceda_observation, caplog -): - """ - Checkout non-failing output and warning when pre-processor fails to provide the - template keys. - """ - - input = "/badc" - expected = {} - - output = iso19115_processor.run(input, pre_processors=[ceda_observation]) - - assert output == expected - assert "URL templating failed" in caplog.text diff --git a/tests/test_baker.py b/tests/test_baker.py new file mode 100644 index 0000000..e8ce39b --- /dev/null +++ b/tests/test_baker.py @@ -0,0 +1,43 @@ +# encoding: utf-8 +""" + +""" +__author__ = "Richard Smith" +__date__ = "22 Oct 2021" +__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" +__license__ = "BSD - see LICENSE file in top-level package directory" +__contact__ = "richard.d.smith@stfc.ac.uk" + +import os + +import pytest + +from stac_generator.core.baker import Recipes + +ROOT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_recipes") + +default_recipe = { + "paths": [], + "type": "item", + "id": [], + "extraction_methods": [], +} + + +@pytest.fixture +def recipes(): + return Recipes(ROOT_PATH) + + +def test_retrieve_posix_description(recipes): + + recipe = recipes.get_description("/a/b/c/d/e") + + assert recipe.paths == ["/a/b/c", "gc://a/b/c"] + + +def test_retrieve_remote_description(recipes): + + recipe = recipes.get_description("gc://a/b/c/d/e") + + assert recipe.paths == ["/a/b/c", "gc://a/b/c"] diff --git a/tests/test_collection_describer.py b/tests/test_collection_describer.py deleted file mode 100644 index 4a3bd00..0000000 --- a/tests/test_collection_describer.py +++ /dev/null @@ -1,49 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "22 Oct 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" - -import os - -import pytest - -from stac_generator.core.baker import Recipes - -ROOT_DESCRIPTIONS = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "test_descriptions" -) - -default_description = { - "paths": [], - "asset": None, - "item": None, - "collection": None, - "categories": [], -} - - -@pytest.fixture -def collection_descriptions(): - print(ROOT_DESCRIPTIONS) - return CollectionDescriptions(ROOT_DESCRIPTIONS) - - -def test_retrieve_posix_description(collection_descriptions): - expected = {**default_description, **{"paths": ["/a/b/c"]}} - - description = collection_descriptions.get_description("/a/b/c/d/e") - - assert description.dict() == expected - - -def test_retrieve_remote_description(collection_descriptions): - expected = {**default_description, **{"paths": ["gc://a/b/c"]}} - - description = collection_descriptions.get_description("gc://a/b/c/d/e") - - assert description.dict() == expected diff --git a/tests/test_descriptions/a.yaml b/tests/test_descriptions/a.yaml deleted file mode 100644 index 5fa2090..0000000 --- a/tests/test_descriptions/a.yaml +++ /dev/null @@ -1,2 +0,0 @@ -paths: - - /a/b/c diff --git a/tests/test_descriptions/b.yaml b/tests/test_descriptions/b.yaml deleted file mode 100644 index 0dece94..0000000 --- a/tests/test_descriptions/b.yaml +++ /dev/null @@ -1,2 +0,0 @@ -paths: - - gc://a/b/c diff --git a/tests/test_elasticsearch_output.py b/tests/test_elasticsearch_output.py deleted file mode 100644 index 4f2a2a1..0000000 --- a/tests/test_elasticsearch_output.py +++ /dev/null @@ -1,36 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "30 Jul 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" - -import pytest - -from stac_generator.plugins.outputs.elasticsearch import ElasticsearchOutput - - -@pytest.fixture -def elasticsearch_backend(): - return ElasticsearchOutput(connection_kwargs={}, index={"name": "test"}) - - -def test_elasticsearch_clean(elasticsearch_backend): - data = {"body": {"bbox": ["-37", "38", "-28", "42"]}} - - expected = { - "body": { - "spatial": { - "bbox": { - "type": "envelope", - "coordinates": [["-37", "42"], ["-28", "38"]], - } - } - } - } - - output = elasticsearch_backend.clean(data) - assert output == expected diff --git a/tests/test_recipes/a.yaml b/tests/test_recipes/a.yaml new file mode 100644 index 0000000..39aa277 --- /dev/null +++ b/tests/test_recipes/a.yaml @@ -0,0 +1,21 @@ +paths: + - /a/b/c + - gc://a/b/c + +type: item + +id: + - method: default + inputs: + defaults: + item_id: $instance_id + +extraction_methods: + - method: regex + inputs: + regex: '\/(?P\w*)\.(?P\w*)\.(?P[\w-]*)\.(?P[\w-]*)\/(?P[\w-]*)\.(?P\w*)\.(?P\w*)\.(?P\w*)\.(?P\w*)\.(?P\w*)' + + - method: remove + inputs: + keys: + - uri diff --git a/tests/test_recipes/b.yaml b/tests/test_recipes/b.yaml new file mode 100644 index 0000000..21e8ef0 --- /dev/null +++ b/tests/test_recipes/b.yaml @@ -0,0 +1,21 @@ +paths: + - /a/b + - gc://a/b + +type: item + +id: + - method: default + inputs: + defaults: + item_id: another_item + +extraction_methods: + - method: regex + inputs: + regex: '\/(?P\w*)\.(?P\w*)\.(?P[\w-]*)\.(?P[\w-]*)\/(?P[\w-]*)\.(?P\w*)\.(?P\w*)\.(?P\w*)\.(?P\w*)\.(?P\w*)' + + - method: remove + inputs: + keys: + - uri diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 201f0a5..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -# encoding: utf-8 -""" - -""" -__author__ = "Richard Smith" -__date__ = "15 Jul 2021" -__copyright__ = "Copyright 2018 United Kingdom Research and Innovation" -__license__ = "BSD - see LICENSE file in top-level package directory" -__contact__ = "richard.d.smith@stfc.ac.uk" - -from stac_generator.core import utils - - -def test_dot2dict_single(): - """ - Check that dot2dict can generate dicts from a single key - """ - value = "test_value" - key = "single_key" - expected = {key: value} - - output = utils.dot2dict(key, value) - - assert output == expected - - -def test_dot2dict_nested(): - """ - Check that dot2dict can generate dicts from a single key - """ - value = "test_value" - key = "single_key.nested" - expected = {"single_key": {"nested": value}} - - output = utils.dot2dict(key, value) - - assert output == expected