From 78016f6b6e01de1b3132e12ebeaf431da61537c2 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Fri, 26 Jan 2024 17:03:09 +1300 Subject: [PATCH] feat!: add new fields to the Collection TDE-985 (#765) * feat: add new fields from LINZ STAC extension to the Collection TDE-985 * fix: tests fails * fix: remove linz STAC extension from Collection * wip * feat: add stac metadata TDE-985 * build: revert pylint change * tests: fix the tests * fix: --geographic_desc should be --geographic_description * fix: argument help message should not contain unecessary - * tests: event_name should not be in all the tests * fix: formatting * fix: should not break if --category is passed as human readable values * refactor: reduce number of dict lookup * fix: location should be geographic_description * feat: change --location to --geographic-description * fix: formatting * fix: unused 'type: ignore' comment * build: fix mypy ignore comment * refactor: check historical imagery based on its category rather than survey number --- scripts/collection_from_items.py | 40 +++--- scripts/stac/imagery/collection.py | 117 +++++++++++------- scripts/stac/imagery/metadata_constants.py | 38 +++--- scripts/stac/tests/collection_test.py | 58 ++++++--- .../stac/tests/generate_description_test.py | 46 +++---- scripts/stac/tests/generate_title_test.py | 88 +++++++------ scripts/stac/tests/item_test.py | 12 +- 7 files changed, 243 insertions(+), 156 deletions(-) diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index 0ecb5d7e9..3644396fc 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -10,12 +10,7 @@ from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri from scripts.logging.time_helper import time_in_ms from scripts.stac.imagery.collection import ImageryCollection -from scripts.stac.imagery.metadata_constants import ( - HUMAN_READABLE_REGIONS, - CollectionTitleMetadata, - ElevationCategories, - ImageryCategories, -) +from scripts.stac.imagery.metadata_constants import DATA_CATEGORIES, HUMAN_READABLE_REGIONS, CollectionMetadata from scripts.stac.imagery.provider import Provider, ProviderRole @@ -29,7 +24,7 @@ def main() -> None: dest="category", help="Dataset category description", required=True, - choices=[type.value for type in ImageryCategories] + [type.value for type in ElevationCategories], + choices=list(DATA_CATEGORIES.keys()) + list(DATA_CATEGORIES.values()), ) parser.add_argument( "--region", @@ -40,7 +35,11 @@ def main() -> None: ) parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True) parser.add_argument( - "--location", dest="location", help="Optional Location of dataset, e.g.- Hutt City", type=str, required=False + "--geographic-description", + dest="geographic_description", + help="Optional Geographic Description of dataset, e.g. Hutt City", + type=str, + required=False, ) parser.add_argument( "--start-date", @@ -56,7 +55,7 @@ def main() -> None: parser.add_argument( "--historic-survey-number", dest="historic_survey_number", - help="Historic Survey Number if Applicable. E.g.- SCN8844", + help="Historic Survey Number if Applicable. E.g. SCN8844", type=str, required=False, ) @@ -92,19 +91,32 @@ def main() -> None: for licensor_name in coalesce_multi_single(arguments.licensor_list, arguments.licensor): providers.append({"name": licensor_name, "roles": [ProviderRole.LICENSOR]}) - title_metadata: CollectionTitleMetadata = { - "category": arguments.category, + # category can also be passed as human readable name (e.g. "Aerial Photos") + # Get the corresponding identifier to simplify the process + category = arguments.category + if not DATA_CATEGORIES.get(category): + for key, value in DATA_CATEGORIES.items(): + if value == category: + category = key + break + + collection_metadata: CollectionMetadata = { + "category": category, "region": arguments.region, "gsd": arguments.gsd, "start_datetime": arguments.start_date, "end_datetime": arguments.end_date, "lifecycle": arguments.lifecycle, - "location": arguments.location, - "event": arguments.event, + "geographic_description": arguments.geographic_description, + "event_name": arguments.event, "historic_survey_number": arguments.historic_survey_number, } - collection = ImageryCollection(title_metadata=title_metadata, collection_id=arguments.collection_id, providers=providers) + collection = ImageryCollection( + metadata=collection_metadata, + collection_id=arguments.collection_id, + providers=providers, + ) if not uri.startswith("s3://"): msg = f"uri is not a s3 path: {uri}" diff --git a/scripts/stac/imagery/collection.py b/scripts/stac/imagery/collection.py index f23053dc5..a67b32f00 100644 --- a/scripts/stac/imagery/collection.py +++ b/scripts/stac/imagery/collection.py @@ -7,10 +7,16 @@ from scripts.files.files_helper import ContentType from scripts.files.fs import write from scripts.stac.imagery.metadata_constants import ( + DATA_CATEGORIES, + DEM, + DSM, HUMAN_READABLE_REGIONS, - CollectionTitleMetadata, - ElevationCategories, - ImageryCategories, + RURAL_AERIAL_PHOTOS, + SATELLITE_IMAGERY, + SCANNED_AERIAL_PHOTOS, + URBAN_AERIAL_PHOTOS, + CollectionMetadata, + MissingMetadataError, SubtypeParameterError, ) from scripts.stac.imagery.provider import Provider, ProviderRole @@ -22,14 +28,14 @@ class ImageryCollection: def __init__( self, - title_metadata: CollectionTitleMetadata, + metadata: CollectionMetadata, collection_id: Optional[str] = None, providers: Optional[List[Provider]] = None, ) -> None: if not collection_id: collection_id = str(ulid.ULID()) - self.title_metadata = title_metadata + self.metadata = metadata self.stac = { "type": "Collection", @@ -40,8 +46,18 @@ def __init__( "license": "CC-BY-4.0", "links": [{"rel": "self", "href": "./collection.json", "type": "application/json"}], "providers": [], + "linz:lifecycle": metadata["lifecycle"], + "linz:geospatial_category": metadata["category"], + "linz:region": metadata["region"], + "linz:security_classification": "unclassified", } + # Optional metadata + if event_name := metadata.get("event_name"): + self.stac["linz:event_name"] = event_name + if geographic_description := metadata.get("geographic_description"): + self.stac["linz:geographic_description"] = geographic_description + # If the providers passed has already a LINZ provider: add its default roles to it has_linz = False if providers: @@ -195,50 +211,59 @@ def write_to(self, destination: str) -> None: def _title(self) -> str: """Generates the title for imagery and elevation datasets. Satellite Imagery / Urban Aerial Photos / Rural Aerial Photos: - [Location / Region if no Location specified] [GSD] [?Event Name] [Data Sub-Type] ([Year(s)]) [?- Preview] + [geographic_description / Region if no geographic_description specified] [GSD] [?Event Name] [Data Sub-Type] + ([Year(s)]) [?- Preview] DEM / DSM: - [Location / Region if no Location specified] [?- Event Name] LiDAR [GSD] [Data Sub-Type] ([Year(s)]) [?- Preview] + [geographic_description / Region if no geographic_description specified] [?- Event Name] LiDAR [GSD] + [Data Sub-Type] ([Year(s)]) [?- Preview] If Historic Survey Number: - [Location / Region if no Location specified] [GSD] [Survey Number] ([Year(s)]) [?- Preview] + [geographic_description / Region if no geographic_description specified] [GSD] [Survey Number] ([Year(s)]) + [?- Preview] Returns: Dataset Title """ # format optional metadata - location = self.title_metadata.get("location") - historic_survey_number = self.title_metadata.get("historic_survey_number") - event = self.title_metadata.get("event") + geographic_description = self.metadata.get("geographic_description") + historic_survey_number = self.metadata.get("historic_survey_number") + event = self.metadata.get("event_name") # format date for metadata - if self.title_metadata["start_datetime"].year == self.title_metadata["end_datetime"].year: - date = str(self.title_metadata["start_datetime"].year) + if self.metadata["start_datetime"].year == self.metadata["end_datetime"].year: + date = str(self.metadata["start_datetime"].year) else: - date = f"{self.title_metadata['start_datetime'].year}-{self.title_metadata['end_datetime'].year}" + date = f"{self.metadata['start_datetime'].year}-{self.metadata['end_datetime'].year}" # determine dataset name - if location: - name = location + if geographic_description: + name = geographic_description else: - name = HUMAN_READABLE_REGIONS[self.title_metadata["region"]] + name = HUMAN_READABLE_REGIONS[self.metadata["region"]] # determine if dataset is preview - if self.title_metadata.get("lifecycle") == "preview": + if self.metadata.get("lifecycle") == "preview": preview = "- Preview" else: preview = None - if historic_survey_number: - return " ".join(f"{name} {self.title_metadata['gsd']} {historic_survey_number} ({date}) {preview or ''}".split()) + if self.metadata["category"] == SCANNED_AERIAL_PHOTOS: + if not historic_survey_number: + raise MissingMetadataError("historic_survey_number") + return " ".join(f"{name} {self.metadata['gsd']} {historic_survey_number} ({date}) {preview or ''}".split()) - if self.title_metadata["category"] in [ImageryCategories.SATELLITE, ImageryCategories.URBAN, ImageryCategories.RURAL]: + if self.metadata["category"] in [ + SATELLITE_IMAGERY, + URBAN_AERIAL_PHOTOS, + RURAL_AERIAL_PHOTOS, + ]: return " ".join( - f"{name} {self.title_metadata['gsd']} {event or ''} {self.title_metadata['category']} ({date}) {preview or ''}".split() # pylint: disable=line-too-long + f"{name} {self.metadata['gsd']} {event or ''} {DATA_CATEGORIES[self.metadata['category']]} ({date}) {preview or ''}".split() # pylint: disable=line-too-long ) - if self.title_metadata["category"] in [ElevationCategories.DEM, ElevationCategories.DSM]: + if self.metadata["category"] in [DEM, DSM]: return " ".join( - f"{name} {self._elevation_title_event(event) or ''} LiDAR {self.title_metadata['gsd']} {self.title_metadata['category']} ({date}) {preview or ''}".split() # pylint: disable=line-too-long + f"{name} {self._elevation_title_event(event) or ''} LiDAR {self.metadata['gsd']} {DATA_CATEGORIES[self.metadata['category']]} ({date}) {preview or ''}".split() # pylint: disable=line-too-long ) - raise SubtypeParameterError(self.title_metadata["category"]) + raise SubtypeParameterError(self.metadata["category"]) def _elevation_title_event(self, event: Optional[str]) -> Optional[str]: if event: @@ -250,7 +275,8 @@ def _description(self) -> str: Urban Aerial Photos / Rural Aerial Photos: Orthophotography within the [Region] region captured in the [Year(s)] flying season. DEM / DSM: - [Digital Surface Model / Digital Elevation Model] within the [region] [?- location] region in [year(s)]. + [Digital Surface Model / Digital Elevation Model] within the [region] + [?- geographic_description] region in [year(s)]. Satellite Imagery: Satellite imagery within the [Region] region captured in [Year(s)]. Historical Imagery: @@ -260,34 +286,37 @@ def _description(self) -> str: Dataset Description """ # format optional metadata - location = self.title_metadata.get("location") - historic_survey_number = self.title_metadata.get("historic_survey_number") - event = self.title_metadata.get("event") + geographic_description = self.metadata.get("geographic_description") + event = self.metadata.get("event_name") # format date for metadata - if self.title_metadata["start_datetime"].year == self.title_metadata["end_datetime"].year: - date = str(self.title_metadata["start_datetime"].year) + if self.metadata["start_datetime"].year == self.metadata["end_datetime"].year: + date = str(self.metadata["start_datetime"].year) else: - date = f"{self.title_metadata['start_datetime'].year}-{self.title_metadata['end_datetime'].year}" + date = f"{self.metadata['start_datetime'].year}-{self.metadata['end_datetime'].year}" - # format location for metadata description - if location: - location = f"- {location}" + # format geographic_description for metadata description + if geographic_description: + geographic_description = f"- {geographic_description}" - region = HUMAN_READABLE_REGIONS[self.title_metadata["region"]] + region = HUMAN_READABLE_REGIONS[self.metadata["region"]] - if historic_survey_number: + if self.metadata["category"] == SCANNED_AERIAL_PHOTOS: desc = f"Scanned aerial imagery within the {region} region captured in {date}" - elif self.title_metadata["category"] == ImageryCategories.SATELLITE: + elif self.metadata["category"] == SATELLITE_IMAGERY: desc = f"Satellite imagery within the {region} region captured in {date}" - elif self.title_metadata["category"] in [ImageryCategories.URBAN, ImageryCategories.RURAL]: + elif self.metadata["category"] in [URBAN_AERIAL_PHOTOS, RURAL_AERIAL_PHOTOS]: desc = f"Orthophotography within the {region} region captured in the {date} flying season" - elif self.title_metadata["category"] == ElevationCategories.DEM: - desc = " ".join(f"Digital Elevation Model within the {region} {location or ''} region in {date}".split()) - elif self.title_metadata["category"] == ElevationCategories.DSM: - desc = " ".join(f"Digital Surface Model within the {region} {location or ''} region in {date}".split()) + elif self.metadata["category"] == DEM: + desc = " ".join( + f"Digital Elevation Model within the {region} {geographic_description or ''} region in {date}".split() + ) + elif self.metadata["category"] == DSM: + desc = " ".join( + f"Digital Surface Model within the {region} {geographic_description or ''} region in {date}".split() + ) else: - raise SubtypeParameterError(self.title_metadata["category"]) + raise SubtypeParameterError(self.metadata["category"]) if event: desc = desc + f", published as a record of the {event} event" diff --git a/scripts/stac/imagery/metadata_constants.py b/scripts/stac/imagery/metadata_constants.py index cb1f6ad99..09a17806e 100644 --- a/scripts/stac/imagery/metadata_constants.py +++ b/scripts/stac/imagery/metadata_constants.py @@ -1,9 +1,8 @@ from datetime import datetime -from enum import Enum from typing import Optional, TypedDict -class CollectionTitleMetadata(TypedDict): +class CollectionMetadata(TypedDict): """ region: Region of Dataset gsd: Dataset Ground Sample Distance @@ -11,7 +10,7 @@ class CollectionTitleMetadata(TypedDict): end_date: Dataset capture end date lifecycle: Dataset status Optional: - location: Optional location of dataset, e.g. Hutt City + geographic_description: Optional geographic_description of dataset, e.g. Hutt City event: Optional details of capture event, e.g. Cyclone Gabrielle historic_survey_number: Optional historic imagery survey number, e.g. SNC88445 """ @@ -22,8 +21,8 @@ class CollectionTitleMetadata(TypedDict): start_datetime: datetime end_datetime: datetime lifecycle: str - location: Optional[str] - event: Optional[str] + geographic_description: Optional[str] + event_name: Optional[str] historic_survey_number: Optional[str] @@ -32,17 +31,28 @@ def __init__(self, category: str) -> None: self.message = f"Unrecognised/Unimplemented Subtype Parameter: {category}" -class ImageryCategories(str, Enum): - SATELLITE = "Satellite Imagery" - URBAN = "Urban Aerial Photos" - RURAL = "Rural Aerial Photos" - AERIAL = "Aerial Photos" - HISTORICAL = "Scanned Aerial Photos" +class MissingMetadataError(Exception): + def __init__(self, metadata: str) -> None: + self.message = f"Missing metadata: {metadata}" -class ElevationCategories(str, Enum): - DEM = "DEM" - DSM = "DSM" +AERIAL_PHOTOS = "aerial-photos" +SCANNED_AERIAL_PHOTOS = "scanned-aerial-photos" +RURAL_AERIAL_PHOTOS = "rural-aerial-photos" +SATELLITE_IMAGERY = "satellite-imagery" +URBAN_AERIAL_PHOTOS = "urban-aerial-photos" +DEM = "dem" +DSM = "dsm" + +DATA_CATEGORIES = { + AERIAL_PHOTOS: "Aerial Photos", + SCANNED_AERIAL_PHOTOS: "Scanned Aerial Photos", + RURAL_AERIAL_PHOTOS: "Rural Aerial Photos", + SATELLITE_IMAGERY: "Satellite Imagery", + URBAN_AERIAL_PHOTOS: "Urban Aerial Photos", + DEM: "DEM", + DSM: "DSM", +} HUMAN_READABLE_REGIONS = { diff --git a/scripts/stac/tests/collection_test.py b/scripts/stac/tests/collection_test.py index 2a320e9ba..379e05cb9 100644 --- a/scripts/stac/tests/collection_test.py +++ b/scripts/stac/tests/collection_test.py @@ -10,48 +10,56 @@ from scripts.files.fs import read from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.item import ImageryItem -from scripts.stac.imagery.metadata_constants import CollectionTitleMetadata +from scripts.stac.imagery.metadata_constants import CollectionMetadata from scripts.stac.imagery.provider import Provider, ProviderRole # pylint: disable=duplicate-code @pytest.fixture(name="metadata", autouse=True) -def setup() -> Generator[CollectionTitleMetadata, None, None]: - metadata: CollectionTitleMetadata = { - "category": "Urban Aerial Photos", +def setup() -> Generator[CollectionMetadata, None, None]: + metadata: CollectionMetadata = { + "category": "urban-aerial-photos", "region": "auckland", "gsd": "0.3m", "start_datetime": datetime(2022, 2, 2), "end_datetime": datetime(2022, 2, 2), "lifecycle": "completed", - "location": None, - "event": None, + "event_name": "Forest assessment", "historic_survey_number": None, + "geographic_description": "Auckland North", } yield metadata -def test_title_description_id_created_on_init(metadata: CollectionTitleMetadata) -> None: +def test_title_description_id_created_on_init(metadata: CollectionMetadata) -> None: collection = ImageryCollection(metadata) - assert collection.stac["title"] == "Auckland 0.3m Urban Aerial Photos (2022)" - assert collection.stac["description"] == "Orthophotography within the Auckland region captured in the 2022 flying season." + assert collection.stac["title"] == "Auckland North 0.3m Forest assessment Urban Aerial Photos (2022)" + assert ( + collection.stac["description"] + == "Orthophotography within the Auckland region captured in the 2022 flying season, published as a record of the Forest assessment event." # pylint: disable=line-too-long + ) assert collection.stac["id"] + assert collection.stac["linz:region"] == "auckland" + assert collection.stac["linz:geographic_description"] == "Auckland North" + assert collection.stac["linz:event_name"] == "Forest assessment" + assert collection.stac["linz:lifecycle"] == "completed" + assert collection.stac["linz:geospatial_category"] == "urban-aerial-photos" -def test_id_parsed_on_init(metadata: CollectionTitleMetadata) -> None: +def test_id_parsed_on_init(metadata: CollectionMetadata) -> None: id_ = "Parsed-Ulid" collection = ImageryCollection(metadata, id_) assert collection.stac["id"] == "Parsed-Ulid" -def test_bbox_updated_from_none(metadata: CollectionTitleMetadata) -> None: +def test_bbox_updated_from_none(metadata: CollectionMetadata) -> None: collection = ImageryCollection(metadata) bbox = [1799667.5, 5815977.0, 1800422.5, 5814986.0] collection.update_spatial_extent(bbox) assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] -def test_bbox_updated_from_existing(metadata: CollectionTitleMetadata) -> None: +def test_bbox_updated_from_existing(metadata: CollectionMetadata) -> None: collection = ImageryCollection(metadata) # init bbox bbox = [174.889641, -41.217532, 174.902344, -41.203521] @@ -63,7 +71,7 @@ def test_bbox_updated_from_existing(metadata: CollectionTitleMetadata) -> None: assert collection.stac["extent"]["spatial"]["bbox"] == [[174.889641, -41.217532, 174.922965, -41.203521]] -def test_interval_updated_from_none(metadata: CollectionTitleMetadata) -> None: +def test_interval_updated_from_none(metadata: CollectionMetadata) -> None: collection = ImageryCollection(metadata) start_datetime = "2021-01-27T00:00:00Z" end_datetime = "2021-01-27T00:00:00Z" @@ -71,7 +79,7 @@ def test_interval_updated_from_none(metadata: CollectionTitleMetadata) -> None: assert collection.stac["extent"]["temporal"]["interval"] == [[start_datetime, end_datetime]] -def test_interval_updated_from_existing(metadata: CollectionTitleMetadata) -> None: +def test_interval_updated_from_existing(metadata: CollectionMetadata) -> None: collection = ImageryCollection(metadata) # init interval start_datetime = "2021-01-27T00:00:00Z" @@ -85,7 +93,7 @@ def test_interval_updated_from_existing(metadata: CollectionTitleMetadata) -> No assert collection.stac["extent"]["temporal"]["interval"] == [["2021-01-27T00:00:00Z", "2021-02-20T00:00:00Z"]] -def test_add_item(mocker, metadata: CollectionTitleMetadata) -> None: # type: ignore +def test_add_item(mocker, metadata: CollectionMetadata) -> None: # type: ignore collection = ImageryCollection(metadata) checksum = "1220cdef68d62fb912110b810e62edc53de07f7a44fb2b310db700e9d9dd58baa6b4" mocker.patch("scripts.stac.util.checksum.multihash_as_hex", return_value=checksum) @@ -107,7 +115,7 @@ def test_add_item(mocker, metadata: CollectionTitleMetadata) -> None: # type: i assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] -def test_write_collection(metadata: CollectionTitleMetadata) -> None: +def test_write_collection(metadata: CollectionMetadata) -> None: target = mkdtemp() collectionObj = ImageryCollection(metadata) collection_target = os.path.join(target, "collection.json") @@ -118,7 +126,7 @@ def test_write_collection(metadata: CollectionTitleMetadata) -> None: assert collection["title"] == collectionObj.stac["title"] -def test_write_collection_special_chars(metadata: CollectionTitleMetadata) -> None: +def test_write_collection_special_chars(metadata: CollectionMetadata) -> None: target = mkdtemp() title = "Manawatū-Whanganui" collectionObj = ImageryCollection(metadata) @@ -131,7 +139,7 @@ def test_write_collection_special_chars(metadata: CollectionTitleMetadata) -> No assert collection["title"] == title -def test_add_providers(metadata: CollectionTitleMetadata) -> None: +def test_add_providers(metadata: CollectionMetadata) -> None: collection = ImageryCollection(metadata) producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]} collection.add_providers([producer]) @@ -139,7 +147,7 @@ def test_add_providers(metadata: CollectionTitleMetadata) -> None: assert {"name": "Maxar", "roles": ["producer"]} in collection.stac["providers"] -def test_default_provider_roles_are_kept(metadata: CollectionTitleMetadata) -> None: +def test_default_provider_roles_are_kept(metadata: CollectionMetadata) -> None: # given we are adding a non default role to the default provider licensor: Provider = {"name": "Toitū Te Whenua Land Information New Zealand", "roles": [ProviderRole.LICENSOR]} producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]} @@ -156,7 +164,7 @@ def test_default_provider_roles_are_kept(metadata: CollectionTitleMetadata) -> N ] -def test_default_provider_is_present(metadata: CollectionTitleMetadata) -> None: +def test_default_provider_is_present(metadata: CollectionMetadata) -> None: # given adding a provider producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]} collection = ImageryCollection(metadata, providers=[producer]) @@ -167,3 +175,13 @@ def test_default_provider_is_present(metadata: CollectionTitleMetadata) -> None: ] # then the new provider is added assert {"name": "Maxar", "roles": ["producer"]} in collection.stac["providers"] + + +def test_event_name_is_present(metadata: CollectionMetadata) -> None: + collection = ImageryCollection(metadata) + assert "Forest assessment" == collection.stac["linz:event_name"] + + +def test_geographic_description_is_present(metadata: CollectionMetadata) -> None: + collection = ImageryCollection(metadata) + assert "Auckland North" == collection.stac["linz:geographic_description"] diff --git a/scripts/stac/tests/generate_description_test.py b/scripts/stac/tests/generate_description_test.py index afb0aa612..28e685e63 100644 --- a/scripts/stac/tests/generate_description_test.py +++ b/scripts/stac/tests/generate_description_test.py @@ -4,83 +4,83 @@ import pytest from scripts.stac.imagery.collection import ImageryCollection -from scripts.stac.imagery.metadata_constants import CollectionTitleMetadata +from scripts.stac.imagery.metadata_constants import CollectionMetadata # pylint: disable=duplicate-code @pytest.fixture(name="metadata", autouse=True) -def setup() -> Generator[Tuple[CollectionTitleMetadata, CollectionTitleMetadata], None, None]: - metadata_auck: CollectionTitleMetadata = { - "category": "Rural Aerial Photos", +def setup() -> Generator[Tuple[CollectionMetadata, CollectionMetadata], None, None]: + metadata_auck: CollectionMetadata = { + "category": "rural-aerial-photos", "region": "auckland", "gsd": "0.3m", "start_datetime": datetime(2023, 1, 1), "end_datetime": datetime(2023, 2, 2), "lifecycle": "completed", - "location": None, - "event": None, + "geographic_description": None, + "event_name": None, "historic_survey_number": None, } - metadata_hb: CollectionTitleMetadata = { - "category": "Rural Aerial Photos", + metadata_hb: CollectionMetadata = { + "category": "rural-aerial-photos", "region": "hawkes-bay", "gsd": "0.3m", "start_datetime": datetime(2023, 1, 1), "end_datetime": datetime(2023, 2, 2), "lifecycle": "completed", - "location": None, - "event": None, + "geographic_description": None, + "event_name": None, "historic_survey_number": None, } yield (metadata_auck, metadata_hb) -def test_generate_description_imagery(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_description_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata collection = ImageryCollection(metadata_auck) description = "Orthophotography within the Auckland region captured in the 2023 flying season." assert collection.stac["description"] == description -def test_generate_description_elevation(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_description_elevation(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "DEM" + metadata_auck["category"] = "dem" collection = ImageryCollection(metadata_auck) description = "Digital Elevation Model within the Auckland region in 2023." assert collection.stac["description"] == description -def test_generate_description_elevation_location_input( - metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata] +def test_generate_description_elevation_geographic_description_input( + metadata: Tuple[CollectionMetadata, CollectionMetadata] ) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "DEM" - metadata_auck["location"] = "Central" + metadata_auck["category"] = "dem" + metadata_auck["geographic_description"] = "Central" collection = ImageryCollection(metadata_auck) description = "Digital Elevation Model within the Auckland - Central region in 2023." assert collection.stac["description"] == description -def test_generate_description_satellite_imagery(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_description_satellite_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "Satellite Imagery" + metadata_auck["category"] = "satellite-imagery" collection = ImageryCollection(metadata_auck) description = "Satellite imagery within the Auckland region captured in 2023." assert collection.stac["description"] == description -def test_generate_description_historic_imagery(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_description_historic_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "Aerial Photos" + metadata_auck["category"] = "scanned-aerial-photos" metadata_auck["historic_survey_number"] = "SNC8844" collection = ImageryCollection(metadata_auck) description = "Scanned aerial imagery within the Auckland region captured in 2023." assert collection.stac["description"] == description -def test_generate_description_event(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_description_event(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: _, metadata_hb = metadata - metadata_hb["event"] = "Cyclone Gabrielle" + metadata_hb["event_name"] = "Cyclone Gabrielle" collection = ImageryCollection(metadata_hb) description = "Orthophotography within the Hawke's Bay region captured in the 2023 flying season, \ published as a record of the Cyclone Gabrielle event." diff --git a/scripts/stac/tests/generate_title_test.py b/scripts/stac/tests/generate_title_test.py index 9f2f24a2a..4832a318c 100644 --- a/scripts/stac/tests/generate_title_test.py +++ b/scripts/stac/tests/generate_title_test.py @@ -4,78 +4,88 @@ import pytest from scripts.stac.imagery.collection import ImageryCollection -from scripts.stac.imagery.metadata_constants import CollectionTitleMetadata +from scripts.stac.imagery.metadata_constants import CollectionMetadata, MissingMetadataError # pylint: disable=duplicate-code @pytest.fixture(name="metadata", autouse=True) -def setup() -> Generator[Tuple[CollectionTitleMetadata, CollectionTitleMetadata], None, None]: - metadata_auck: CollectionTitleMetadata = { - "category": "Rural Aerial Photos", +def setup() -> Generator[Tuple[CollectionMetadata, CollectionMetadata], None, None]: + metadata_auck: CollectionMetadata = { + "category": "rural-aerial-photos", "region": "auckland", "gsd": "0.3m", "start_datetime": datetime(2023, 1, 1), "end_datetime": datetime(2023, 2, 2), "lifecycle": "completed", - "location": None, - "event": None, + "event_name": None, "historic_survey_number": None, + "geographic_description": None, } - metadata_hb: CollectionTitleMetadata = { - "category": "Rural Aerial Photos", + metadata_hb: CollectionMetadata = { + "category": "rural-aerial-photos", "region": "hawkes-bay", "gsd": "0.3m", "start_datetime": datetime(2023, 1, 1), "end_datetime": datetime(2023, 2, 2), "lifecycle": "completed", - "location": None, - "event": None, + "event_name": None, "historic_survey_number": None, + "geographic_description": None, } yield (metadata_auck, metadata_hb) -def test_generate_imagery_title(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata title = "Auckland 0.3m Rural Aerial Photos (2023)" collection = ImageryCollection(metadata_auck) assert collection.stac["title"] == title -def test_generate_dem_title(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_dem_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "DEM" + metadata_auck["category"] = "dem" collection = ImageryCollection(metadata_auck) title = "Auckland LiDAR 0.3m DEM (2023)" assert collection.stac["title"] == title -def test_generate_dsm_title(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_dsm_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "DSM" + metadata_auck["category"] = "dsm" collection = ImageryCollection(metadata_auck) title = "Auckland LiDAR 0.3m DSM (2023)" assert collection.stac["title"] == title -def test_generate_satellite_imagery_title(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_satellite_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "Satellite Imagery" + metadata_auck["category"] = "satellite-imagery" collection = ImageryCollection(metadata_auck) title = "Auckland 0.3m Satellite Imagery (2023)" assert collection.stac["title"] == title -def test_generate_historic_imagery_title(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_historic_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: title = "Auckland 0.3m SNC8844 (2023)" metadata_auck, _ = metadata - metadata_auck["category"] = "Aerial Photos" + metadata_auck["category"] = "scanned-aerial-photos" metadata_auck["historic_survey_number"] = "SNC8844" collection = ImageryCollection(metadata_auck) assert collection.stac["title"] == title -def test_generate_title_long_date(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_historic_imagery_title_missing_number(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: + metadata_auck, _ = metadata + metadata_auck["category"] = "scanned-aerial-photos" + metadata_auck["historic_survey_number"] = None + with pytest.raises(MissingMetadataError) as excinfo: + ImageryCollection(metadata_auck) + + assert "historic_survey_number" in str(excinfo.value) + + +def test_generate_title_long_date(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["end_datetime"] = datetime(2024, 1, 1) collection = ImageryCollection(metadata_auck) @@ -83,53 +93,61 @@ def test_generate_title_long_date(metadata: Tuple[CollectionTitleMetadata, Colle assert collection.stac["title"] == title -def test_generate_title_location(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_title_geographic_description(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["location"] = "Ponsonby" + metadata_auck["geographic_description"] = "Ponsonby" collection = ImageryCollection(metadata_auck) title = "Ponsonby 0.3m Rural Aerial Photos (2023)" assert collection.stac["title"] == title -def test_generate_title_event_imagery(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_title_event_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: _, metadata_hb = metadata - metadata_hb["event"] = "Cyclone Gabrielle" + metadata_hb["event_name"] = "Cyclone Gabrielle" collection = ImageryCollection(metadata_hb) title = "Hawke's Bay 0.3m Cyclone Gabrielle Rural Aerial Photos (2023)" assert collection.stac["title"] == title -def test_generate_title_event_elevation(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_title_event_elevation(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: _, metadata_hb = metadata - metadata_hb["category"] = "DSM" - metadata_hb["event"] = "Cyclone Gabrielle" + metadata_hb["category"] = "dsm" + metadata_hb["event_name"] = "Cyclone Gabrielle" collection = ImageryCollection(metadata_hb) title = "Hawke's Bay - Cyclone Gabrielle LiDAR 0.3m DSM (2023)" assert collection.stac["title"] == title -def test_generate_title_event_satellite_imagery(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_title_event_satellite_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: _, metadata_hb = metadata - metadata_hb["category"] = "Satellite Imagery" - metadata_hb["event"] = "Cyclone Gabrielle" + metadata_hb["category"] = "satellite-imagery" + metadata_hb["event_name"] = "Cyclone Gabrielle" collection = ImageryCollection(metadata_hb) title = "Hawke's Bay 0.3m Cyclone Gabrielle Satellite Imagery (2023)" assert collection.stac["title"] == title -def test_generate_dsm_title_preview(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_dsm_title_preview(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["category"] = "DSM" + metadata_auck["category"] = "dsm" metadata_auck["lifecycle"] = "preview" collection = ImageryCollection(metadata_auck) title = "Auckland LiDAR 0.3m DSM (2023) - Preview" assert collection.stac["title"] == title -def test_generate_imagery_title_empty_optional_str(metadata: Tuple[CollectionTitleMetadata, CollectionTitleMetadata]) -> None: +def test_generate_imagery_title_empty_optional_str(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - metadata_auck["location"] = "" - metadata_auck["event"] = "" + metadata_auck["geographic_description"] = "" + metadata_auck["event_name"] = "" collection = ImageryCollection(metadata_auck) title = "Auckland 0.3m Rural Aerial Photos (2023)" assert collection.stac["title"] == title + + +def test_generate_imagery_title_with_event(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: + metadata_auck, _ = metadata + metadata_auck["event_name"] = "Forest assessment" + collection = ImageryCollection(metadata_auck) + title = "Auckland 0.3m Forest assessment Rural Aerial Photos (2023)" + assert collection.stac["title"] == title diff --git a/scripts/stac/tests/item_test.py b/scripts/stac/tests/item_test.py index 501f2ba16..a7d8ef827 100644 --- a/scripts/stac/tests/item_test.py +++ b/scripts/stac/tests/item_test.py @@ -3,7 +3,7 @@ from scripts.files.files_helper import get_file_name_from_path from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.item import ImageryItem -from scripts.stac.imagery.metadata_constants import CollectionTitleMetadata +from scripts.stac.imagery.metadata_constants import CollectionMetadata def test_imagery_stac_item(mocker) -> None: # type: ignore @@ -38,19 +38,19 @@ def test_imagery_stac_item(mocker) -> None: # type: ignore # pylint: disable=duplicate-code def test_imagery_add_collection(mocker) -> None: # type: ignore - metadata: CollectionTitleMetadata = { - "category": "Urban Aerial Photos", + metadata: CollectionMetadata = { + "category": "urban-aerial-photos", "region": "auckland", "gsd": "0.3m", "start_datetime": datetime(2022, 2, 2), "end_datetime": datetime(2022, 2, 2), "lifecycle": "completed", - "location": None, - "event": None, + "event_name": None, "historic_survey_number": None, + "geographic_description": None, } ulid = "fake_ulid" - collection = ImageryCollection(title_metadata=metadata, collection_id=ulid) + collection = ImageryCollection(metadata=metadata, collection_id=ulid) path = "./test/BR34_5000_0302.tiff" id_ = get_file_name_from_path(path)