diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index 1731457e0..37328fc1b 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -9,6 +9,7 @@ from linz_logger import get_log from scripts.cli.cli_helper import coalesce_multi_single, valid_date +from scripts.datetimes import utc_now from scripts.files.files_helper import SUFFIX_FOOTPRINT, SUFFIX_JSON from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_files_in_uri from scripts.logging.time_helper import time_in_ms @@ -108,6 +109,7 @@ def main() -> None: collection = ImageryCollection( metadata=collection_metadata, + now=utc_now, collection_id=arguments.collection_id, providers=providers, ) diff --git a/scripts/stac/imagery/collection.py b/scripts/stac/imagery/collection.py index 6c5232370..2e892f556 100644 --- a/scripts/stac/imagery/collection.py +++ b/scripts/stac/imagery/collection.py @@ -1,5 +1,6 @@ import os -from typing import Any, Dict, List, Optional +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional import shapely.ops import ulid @@ -36,6 +37,7 @@ class ImageryCollection: def __init__( self, metadata: CollectionMetadata, + now: Callable[[], datetime], collection_id: Optional[str] = None, providers: Optional[List[Provider]] = None, ) -> None: @@ -44,6 +46,7 @@ def __init__( self.metadata = metadata + now_string = format_rfc_3339_datetime_string(now()) self.stac = { "type": "Collection", "stac_version": STAC_VERSION, @@ -57,6 +60,8 @@ def __init__( "linz:geospatial_category": metadata["category"], "linz:region": metadata["region"], "linz:security_classification": "unclassified", + "created": now_string, + "updated": now_string, } # Optional metadata diff --git a/scripts/stac/imagery/tests/collection_test.py b/scripts/stac/imagery/tests/collection_test.py index ef12b5720..6c35efe46 100644 --- a/scripts/stac/imagery/tests/collection_test.py +++ b/scripts/stac/imagery/tests/collection_test.py @@ -38,7 +38,7 @@ def setup() -> Generator[CollectionMetadata, None, None]: def test_title_description_id_created_on_init(metadata: CollectionMetadata, subtests: SubTests) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) with subtests.test(): assert collection.stac["title"] == "Auckland North Forest Assessment 0.3m Urban Aerial Photos (2022)" @@ -69,19 +69,19 @@ def test_title_description_id_created_on_init(metadata: CollectionMetadata, subt def test_id_parsed_on_init(metadata: CollectionMetadata) -> None: id_ = "Parsed-Ulid" - collection = ImageryCollection(metadata, id_) + collection = ImageryCollection(metadata, any_epoch_datetime, id_) assert collection.stac["id"] == "Parsed-Ulid" def test_bbox_updated_from_none(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) bbox = [1799667.5, 5815977.0, 1800422.5, 5814986.0] collection.update_spatial_extent(bbox) assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] def test_bbox_updated_from_existing(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) # init bbox bbox = [174.889641, -41.217532, 174.902344, -41.203521] collection.update_spatial_extent(bbox) @@ -93,7 +93,7 @@ def test_bbox_updated_from_existing(metadata: CollectionMetadata) -> None: def test_interval_updated_from_none(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) start_datetime = "2021-01-27T00:00:00Z" end_datetime = "2021-01-27T00:00:00Z" collection.update_temporal_extent(start_datetime, end_datetime) @@ -101,7 +101,7 @@ def test_interval_updated_from_none(metadata: CollectionMetadata) -> None: def test_interval_updated_from_existing(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) # init interval start_datetime = "2021-01-27T00:00:00Z" end_datetime = "2021-01-27T00:00:00Z" @@ -122,12 +122,13 @@ def func() -> datetime: def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: - collection = ImageryCollection(metadata) + now = any_epoch_datetime() + now_function = fixed_now_function(now) + collection = ImageryCollection(metadata, now_function) item_file_path = "./scripts/tests/data/empty.tiff" modified_datetime = datetime(2001, 2, 3, hour=4, minute=5, second=6, tzinfo=timezone.utc) os.utime(item_file_path, times=(any_epoch_datetime().timestamp(), modified_datetime.timestamp())) - now = any_epoch_datetime() - item = ImageryItem("BR34_5000_0304", item_file_path, fixed_now_function(now)) + item = ImageryItem("BR34_5000_0304", item_file_path, now_function) geometry = { "type": "Polygon", "coordinates": [[1799667.5, 5815977.0], [1800422.5, 5815977.0], [1800422.5, 5814986.0], [1799667.5, 5814986.0]], @@ -159,6 +160,9 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] for property_name in ["created", "updated"]: + with subtests.test(msg=f"collection {property_name}"): + assert collection.stac[property_name] == format_rfc_3339_datetime_string(now) + with subtests.test(msg=f"item properties.{property_name}"): assert item.stac["properties"][property_name] == format_rfc_3339_datetime_string(now) @@ -168,7 +172,7 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: def test_write_collection(metadata: CollectionMetadata) -> None: target = mkdtemp() - collectionObj = ImageryCollection(metadata) + collectionObj = ImageryCollection(metadata, any_epoch_datetime) collection_target = os.path.join(target, "collection.json") collectionObj.write_to(collection_target) collection = json.loads(read(collection_target)) @@ -180,7 +184,7 @@ def test_write_collection(metadata: CollectionMetadata) -> None: def test_write_collection_special_chars(metadata: CollectionMetadata) -> None: target = mkdtemp() title = "Manawatū-Whanganui" - collectionObj = ImageryCollection(metadata) + collectionObj = ImageryCollection(metadata, any_epoch_datetime) collectionObj.stac["title"] = title collection_target = os.path.join(target, "collection.json") collectionObj.write_to(collection_target) @@ -191,7 +195,7 @@ def test_write_collection_special_chars(metadata: CollectionMetadata) -> None: def test_add_providers(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]} collection.add_providers([producer]) @@ -202,7 +206,7 @@ def test_default_provider_roles_are_kept(metadata: CollectionMetadata, subtests: # given we are adding a non default role to the default provider licensor: Provider = {"name": "Toitū Te Whenua Land Information New Zealand", "roles": [ProviderRole.LICENSOR]} producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]} - collection = ImageryCollection(metadata, providers=[producer, licensor]) + collection = ImageryCollection(metadata, any_epoch_datetime, providers=[producer, licensor]) with subtests.test(msg="it adds the non default role to the existing default role list"): assert { @@ -219,7 +223,7 @@ def test_default_provider_roles_are_kept(metadata: CollectionMetadata, subtests: def test_default_provider_is_present(metadata: CollectionMetadata, subtests: SubTests) -> None: # given adding a provider producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]} - collection = ImageryCollection(metadata, providers=[producer]) + collection = ImageryCollection(metadata, any_epoch_datetime, providers=[producer]) with subtests.test(msg="the default provider is still present"): assert {"name": "Toitū Te Whenua Land Information New Zealand", "roles": ["host", "processor"]} in collection.stac[ @@ -235,7 +239,7 @@ def test_capture_area_added(metadata: CollectionMetadata, subtests: SubTests) -> . Once we start using geos 3.12 in CI we can delete the values for 3.11 below. """ - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) file_name = "capture-area.geojson" polygons = [] @@ -321,10 +325,10 @@ def test_capture_area_added(metadata: CollectionMetadata, subtests: SubTests) -> def test_event_name_is_present(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) assert "Forest Assessment" == collection.stac["linz:event_name"] def test_geographic_description_is_present(metadata: CollectionMetadata) -> None: - collection = ImageryCollection(metadata) + collection = ImageryCollection(metadata, any_epoch_datetime) assert "Auckland North Forest Assessment" == collection.stac["linz:geographic_description"] diff --git a/scripts/stac/imagery/tests/generate_description_test.py b/scripts/stac/imagery/tests/generate_description_test.py index 58894fbdb..31af04bcd 100644 --- a/scripts/stac/imagery/tests/generate_description_test.py +++ b/scripts/stac/imagery/tests/generate_description_test.py @@ -5,6 +5,7 @@ from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.metadata_constants import CollectionMetadata +from scripts.tests.datetimes_test import any_epoch_datetime # pylint: disable=duplicate-code @@ -37,7 +38,7 @@ def setup() -> Generator[Tuple[CollectionMetadata, CollectionMetadata], None, No def test_generate_description_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) description = "Orthophotography within the Auckland region captured in the 2023 flying season." assert collection.stac["description"] == description @@ -45,7 +46,7 @@ def test_generate_description_imagery(metadata: Tuple[CollectionMetadata, Collec def test_generate_description_elevation(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["category"] = "dem" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) description = "Digital Elevation Model within the Auckland region captured in 2023." assert collection.stac["description"] == description @@ -56,7 +57,7 @@ def test_generate_description_elevation_geographic_description_input( metadata_auck, _ = metadata metadata_auck["category"] = "dem" metadata_auck["geographic_description"] = "Central" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) description = "Digital Elevation Model within the Auckland region captured in 2023." assert collection.stac["description"] == description @@ -64,7 +65,7 @@ def test_generate_description_elevation_geographic_description_input( def test_generate_description_satellite_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["category"] = "satellite-imagery" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) description = "Satellite imagery within the Auckland region captured in 2023." assert collection.stac["description"] == description @@ -73,7 +74,7 @@ def test_generate_description_historic_imagery(metadata: Tuple[CollectionMetadat metadata_auck, _ = metadata metadata_auck["category"] = "scanned-aerial-photos" metadata_auck["historic_survey_number"] = "SNC8844" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) description = "Scanned aerial imagery within the Auckland region captured in 2023." assert collection.stac["description"] == description @@ -81,7 +82,7 @@ def test_generate_description_historic_imagery(metadata: Tuple[CollectionMetadat def test_generate_description_event(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: _, metadata_hb = metadata metadata_hb["event_name"] = "Cyclone Gabrielle" - collection = ImageryCollection(metadata_hb) + collection = ImageryCollection(metadata_hb, any_epoch_datetime) description = "Orthophotography within the Hawke's Bay region captured in the 2023 flying season, \ published as a record of the Cyclone Gabrielle event." assert collection.stac["description"] == description diff --git a/scripts/stac/imagery/tests/generate_title_test.py b/scripts/stac/imagery/tests/generate_title_test.py index e034fa890..2b1a10e10 100644 --- a/scripts/stac/imagery/tests/generate_title_test.py +++ b/scripts/stac/imagery/tests/generate_title_test.py @@ -5,6 +5,7 @@ from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.metadata_constants import CollectionMetadata, MissingMetadataError +from scripts.tests.datetimes_test import any_epoch_datetime # pylint: disable=duplicate-code @@ -38,14 +39,14 @@ def setup() -> Generator[Tuple[CollectionMetadata, CollectionMetadata], None, No def test_generate_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata title = "Auckland 0.3m Rural Aerial Photos (2023)" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) assert collection.stac["title"] == title def test_generate_dem_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["category"] = "dem" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland LiDAR 0.3m DEM (2023)" assert collection.stac["title"] == title @@ -53,7 +54,7 @@ def test_generate_dem_title(metadata: Tuple[CollectionMetadata, CollectionMetada def test_generate_dsm_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["category"] = "dsm" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland LiDAR 0.3m DSM (2023)" assert collection.stac["title"] == title @@ -61,7 +62,7 @@ def test_generate_dsm_title(metadata: Tuple[CollectionMetadata, CollectionMetada def test_generate_satellite_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["category"] = "satellite-imagery" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland 0.3m Satellite Imagery (2023)" assert collection.stac["title"] == title @@ -71,7 +72,7 @@ def test_generate_historic_imagery_title(metadata: Tuple[CollectionMetadata, Col metadata_auck, _ = metadata metadata_auck["category"] = "scanned-aerial-photos" metadata_auck["historic_survey_number"] = "SNC8844" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) assert collection.stac["title"] == title @@ -80,7 +81,7 @@ def test_generate_historic_imagery_title_missing_number(metadata: Tuple[Collecti metadata_auck["category"] = "scanned-aerial-photos" metadata_auck["historic_survey_number"] = None with pytest.raises(MissingMetadataError) as excinfo: - ImageryCollection(metadata_auck) + ImageryCollection(metadata_auck, any_epoch_datetime) assert "historic_survey_number" in str(excinfo.value) @@ -88,7 +89,7 @@ def test_generate_historic_imagery_title_missing_number(metadata: Tuple[Collecti def test_generate_title_long_date(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["end_datetime"] = datetime(2024, 1, 1) - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland 0.3m Rural Aerial Photos (2023-2024)" assert collection.stac["title"] == title @@ -96,7 +97,7 @@ def test_generate_title_long_date(metadata: Tuple[CollectionMetadata, Collection def test_generate_title_geographic_description(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: metadata_auck, _ = metadata metadata_auck["geographic_description"] = "Ponsonby" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Ponsonby 0.3m Rural Aerial Photos (2023)" assert collection.stac["title"] == title @@ -105,7 +106,7 @@ def test_generate_title_event_imagery(metadata: Tuple[CollectionMetadata, Collec _, metadata_hb = metadata metadata_hb["geographic_description"] = "Hawke's Bay Cyclone Gabrielle" metadata_hb["event_name"] = "Cyclone Gabrielle" - collection = ImageryCollection(metadata_hb) + collection = ImageryCollection(metadata_hb, any_epoch_datetime) title = "Hawke's Bay Cyclone Gabrielle 0.3m Rural Aerial Photos (2023)" assert collection.stac["title"] == title @@ -115,7 +116,7 @@ def test_generate_title_event_elevation(metadata: Tuple[CollectionMetadata, Coll metadata_hb["category"] = "dsm" metadata_hb["geographic_description"] = "Hawke's Bay Cyclone Gabrielle" metadata_hb["event_name"] = "Cyclone Gabrielle" - collection = ImageryCollection(metadata_hb) + collection = ImageryCollection(metadata_hb, any_epoch_datetime) title = "Hawke's Bay - Hawke's Bay Cyclone Gabrielle LiDAR 0.3m DSM (2023)" assert collection.stac["title"] == title @@ -125,7 +126,7 @@ def test_generate_title_event_satellite_imagery(metadata: Tuple[CollectionMetada metadata_hb["category"] = "satellite-imagery" metadata_hb["geographic_description"] = "Hawke's Bay Cyclone Gabrielle" metadata_hb["event_name"] = "Cyclone Gabrielle" - collection = ImageryCollection(metadata_hb) + collection = ImageryCollection(metadata_hb, any_epoch_datetime) title = "Hawke's Bay Cyclone Gabrielle 0.3m Satellite Imagery (2023)" assert collection.stac["title"] == title @@ -134,7 +135,7 @@ def test_generate_dsm_title_preview(metadata: Tuple[CollectionMetadata, Collecti metadata_auck, _ = metadata metadata_auck["category"] = "dsm" metadata_auck["lifecycle"] = "preview" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland LiDAR 0.3m DSM (2023) - Preview" assert collection.stac["title"] == title @@ -142,7 +143,7 @@ def test_generate_dsm_title_preview(metadata: Tuple[CollectionMetadata, Collecti def test_generate_imagery_title_draft(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None: _, metadata_hb = metadata metadata_hb["lifecycle"] = "ongoing" - collection = ImageryCollection(metadata_hb) + collection = ImageryCollection(metadata_hb, any_epoch_datetime) title = "Hawke's Bay 0.3m Rural Aerial Photos (2023) - Draft" assert collection.stac["title"] == title @@ -151,7 +152,7 @@ def test_generate_imagery_title_empty_optional_str(metadata: Tuple[CollectionMet metadata_auck, _ = metadata metadata_auck["geographic_description"] = "" metadata_auck["event_name"] = "" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland 0.3m Rural Aerial Photos (2023)" assert collection.stac["title"] == title @@ -160,6 +161,6 @@ def test_generate_imagery_title_with_event(metadata: Tuple[CollectionMetadata, C metadata_auck, _ = metadata metadata_auck["geographic_description"] = "Auckland Forest Assessment" metadata_auck["event_name"] = "Forest Assessment" - collection = ImageryCollection(metadata_auck) + collection = ImageryCollection(metadata_auck, any_epoch_datetime) title = "Auckland Forest Assessment 0.3m Rural Aerial Photos (2023)" assert collection.stac["title"] == title diff --git a/scripts/stac/imagery/tests/item_test.py b/scripts/stac/imagery/tests/item_test.py index e180ccd4b..5ec7a65fc 100644 --- a/scripts/stac/imagery/tests/item_test.py +++ b/scripts/stac/imagery/tests/item_test.py @@ -73,7 +73,7 @@ def test_imagery_add_collection(mocker: MockerFixture, subtests: SubTests) -> No "geographic_description": None, } ulid = "fake_ulid" - collection = ImageryCollection(metadata=metadata, collection_id=ulid) + collection = ImageryCollection(metadata=metadata, now=any_epoch_datetime, collection_id=ulid) path = "./scripts/tests/data/empty.tiff" id_ = get_file_name_from_path(path)