feat: Add created/updated properties to collections TDE-1147 (#981)

These fields are not part of core STAC, and are not yet in the TTW LINZ extension. This has been relegated to a separate story <https://toitutewhenua.atlassian.net/browse/TDE-1192>. #### Motivation Make it easier for end users to determine when a dataset was created/updated by providing these properties in the top level STAC document. #### Checklist - [x] Tests updated - [ ] Docs updated (N/A; `collection.json` structure is not documented in this repository) - [x] Issue linked in Title
linz · Jun 9, 2024 · 00f4570 · 00f4570
1 parent 37ee07f
commit 00f4570
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 40 deletions.
diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py
@@ -9,6 +9,7 @@
 from linz_logger import get_log
 
 from scripts.cli.cli_helper import coalesce_multi_single, valid_date
+from scripts.datetimes import utc_now
 from scripts.files.files_helper import SUFFIX_FOOTPRINT, SUFFIX_JSON
 from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_files_in_uri
 from scripts.logging.time_helper import time_in_ms
@@ -108,6 +109,7 @@ def main() -> None:
 
     collection = ImageryCollection(
         metadata=collection_metadata,
+        now=utc_now,
         collection_id=arguments.collection_id,
         providers=providers,
     )

diff --git a/scripts/stac/imagery/collection.py b/scripts/stac/imagery/collection.py
@@ -1,5 +1,6 @@
 import os
-from typing import Any, Dict, List, Optional
+from datetime import datetime
+from typing import Any, Callable, Dict, List, Optional
 
 import shapely.ops
 import ulid
@@ -36,6 +37,7 @@ class ImageryCollection:
     def __init__(
         self,
         metadata: CollectionMetadata,
+        now: Callable[[], datetime],
         collection_id: Optional[str] = None,
         providers: Optional[List[Provider]] = None,
     ) -> None:
@@ -44,6 +46,7 @@ def __init__(
 
         self.metadata = metadata
 
+        now_string = format_rfc_3339_datetime_string(now())
         self.stac = {
             "type": "Collection",
             "stac_version": STAC_VERSION,
@@ -57,6 +60,8 @@ def __init__(
             "linz:geospatial_category": metadata["category"],
             "linz:region": metadata["region"],
             "linz:security_classification": "unclassified",
+            "created": now_string,
+            "updated": now_string,
         }
 
         # Optional metadata

diff --git a/scripts/stac/imagery/tests/collection_test.py b/scripts/stac/imagery/tests/collection_test.py
@@ -38,7 +38,7 @@ def setup() -> Generator[CollectionMetadata, None, None]:
 
 
 def test_title_description_id_created_on_init(metadata: CollectionMetadata, subtests: SubTests) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     with subtests.test():
         assert collection.stac["title"] == "Auckland North Forest Assessment 0.3m Urban Aerial Photos (2022)"
 
@@ -69,19 +69,19 @@ def test_title_description_id_created_on_init(metadata: CollectionMetadata, subt
 
 def test_id_parsed_on_init(metadata: CollectionMetadata) -> None:
     id_ = "Parsed-Ulid"
-    collection = ImageryCollection(metadata, id_)
+    collection = ImageryCollection(metadata, any_epoch_datetime, id_)
     assert collection.stac["id"] == "Parsed-Ulid"
 
 
 def test_bbox_updated_from_none(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     bbox = [1799667.5, 5815977.0, 1800422.5, 5814986.0]
     collection.update_spatial_extent(bbox)
     assert collection.stac["extent"]["spatial"]["bbox"] == [bbox]
 
 
 def test_bbox_updated_from_existing(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     # init bbox
     bbox = [174.889641, -41.217532, 174.902344, -41.203521]
     collection.update_spatial_extent(bbox)
@@ -93,15 +93,15 @@ def test_bbox_updated_from_existing(metadata: CollectionMetadata) -> None:
 
 
 def test_interval_updated_from_none(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     start_datetime = "2021-01-27T00:00:00Z"
     end_datetime = "2021-01-27T00:00:00Z"
     collection.update_temporal_extent(start_datetime, end_datetime)
     assert collection.stac["extent"]["temporal"]["interval"] == [[start_datetime, end_datetime]]
 
 
 def test_interval_updated_from_existing(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     # init interval
     start_datetime = "2021-01-27T00:00:00Z"
     end_datetime = "2021-01-27T00:00:00Z"
@@ -122,12 +122,13 @@ def func() -> datetime:
 
 
 def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None:
-    collection = ImageryCollection(metadata)
+    now = any_epoch_datetime()
+    now_function = fixed_now_function(now)
+    collection = ImageryCollection(metadata, now_function)
     item_file_path = "./scripts/tests/data/empty.tiff"
     modified_datetime = datetime(2001, 2, 3, hour=4, minute=5, second=6, tzinfo=timezone.utc)
     os.utime(item_file_path, times=(any_epoch_datetime().timestamp(), modified_datetime.timestamp()))
-    now = any_epoch_datetime()
-    item = ImageryItem("BR34_5000_0304", item_file_path, fixed_now_function(now))
+    item = ImageryItem("BR34_5000_0304", item_file_path, now_function)
     geometry = {
         "type": "Polygon",
         "coordinates": [[1799667.5, 5815977.0], [1800422.5, 5815977.0], [1800422.5, 5814986.0], [1799667.5, 5814986.0]],
@@ -159,6 +160,9 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None:
         assert collection.stac["extent"]["spatial"]["bbox"] == [bbox]
 
     for property_name in ["created", "updated"]:
+        with subtests.test(msg=f"collection {property_name}"):
+            assert collection.stac[property_name] == format_rfc_3339_datetime_string(now)
+
         with subtests.test(msg=f"item properties.{property_name}"):
             assert item.stac["properties"][property_name] == format_rfc_3339_datetime_string(now)
 
@@ -168,7 +172,7 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None:
 
 def test_write_collection(metadata: CollectionMetadata) -> None:
     target = mkdtemp()
-    collectionObj = ImageryCollection(metadata)
+    collectionObj = ImageryCollection(metadata, any_epoch_datetime)
     collection_target = os.path.join(target, "collection.json")
     collectionObj.write_to(collection_target)
     collection = json.loads(read(collection_target))
@@ -180,7 +184,7 @@ def test_write_collection(metadata: CollectionMetadata) -> None:
 def test_write_collection_special_chars(metadata: CollectionMetadata) -> None:
     target = mkdtemp()
     title = "Manawatū-Whanganui"
-    collectionObj = ImageryCollection(metadata)
+    collectionObj = ImageryCollection(metadata, any_epoch_datetime)
     collectionObj.stac["title"] = title
     collection_target = os.path.join(target, "collection.json")
     collectionObj.write_to(collection_target)
@@ -191,7 +195,7 @@ def test_write_collection_special_chars(metadata: CollectionMetadata) -> None:
 
 
 def test_add_providers(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]}
     collection.add_providers([producer])
 
@@ -202,7 +206,7 @@ def test_default_provider_roles_are_kept(metadata: CollectionMetadata, subtests:
     # given we are adding a non default role to the default provider
     licensor: Provider = {"name": "Toitū Te Whenua Land Information New Zealand", "roles": [ProviderRole.LICENSOR]}
     producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]}
-    collection = ImageryCollection(metadata, providers=[producer, licensor])
+    collection = ImageryCollection(metadata, any_epoch_datetime, providers=[producer, licensor])
 
     with subtests.test(msg="it adds the non default role to the existing default role list"):
         assert {
@@ -219,7 +223,7 @@ def test_default_provider_roles_are_kept(metadata: CollectionMetadata, subtests:
 def test_default_provider_is_present(metadata: CollectionMetadata, subtests: SubTests) -> None:
     # given adding a provider
     producer: Provider = {"name": "Maxar", "roles": [ProviderRole.PRODUCER]}
-    collection = ImageryCollection(metadata, providers=[producer])
+    collection = ImageryCollection(metadata, any_epoch_datetime, providers=[producer])
 
     with subtests.test(msg="the default provider is still present"):
         assert {"name": "Toitū Te Whenua Land Information New Zealand", "roles": ["host", "processor"]} in collection.stac[
@@ -235,7 +239,7 @@ def test_capture_area_added(metadata: CollectionMetadata, subtests: SubTests) ->
     <https://github.com/libgeos/geos/pull/718>. Once we start using geos 3.12 in CI we can delete the values for 3.11
     below.
     """
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     file_name = "capture-area.geojson"
 
     polygons = []
@@ -321,10 +325,10 @@ def test_capture_area_added(metadata: CollectionMetadata, subtests: SubTests) ->
 
 
 def test_event_name_is_present(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     assert "Forest Assessment" == collection.stac["linz:event_name"]
 
 
 def test_geographic_description_is_present(metadata: CollectionMetadata) -> None:
-    collection = ImageryCollection(metadata)
+    collection = ImageryCollection(metadata, any_epoch_datetime)
     assert "Auckland North Forest Assessment" == collection.stac["linz:geographic_description"]
diff --git a/scripts/stac/imagery/tests/generate_description_test.py b/scripts/stac/imagery/tests/generate_description_test.py
@@ -5,6 +5,7 @@
 
 from scripts.stac.imagery.collection import ImageryCollection
 from scripts.stac.imagery.metadata_constants import CollectionMetadata
+from scripts.tests.datetimes_test import any_epoch_datetime
 
 
 # pylint: disable=duplicate-code
@@ -37,15 +38,15 @@ def setup() -> Generator[Tuple[CollectionMetadata, CollectionMetadata], None, No
 
 def test_generate_description_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     description = "Orthophotography within the Auckland region captured in the 2023 flying season."
     assert collection.stac["description"] == description
 
 
 def test_generate_description_elevation(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["category"] = "dem"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     description = "Digital Elevation Model within the Auckland region captured in 2023."
     assert collection.stac["description"] == description
 
@@ -56,15 +57,15 @@ def test_generate_description_elevation_geographic_description_input(
     metadata_auck, _ = metadata
     metadata_auck["category"] = "dem"
     metadata_auck["geographic_description"] = "Central"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     description = "Digital Elevation Model within the Auckland region captured in 2023."
     assert collection.stac["description"] == description
 
 
 def test_generate_description_satellite_imagery(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["category"] = "satellite-imagery"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     description = "Satellite imagery within the Auckland region captured in 2023."
     assert collection.stac["description"] == description
 
@@ -73,15 +74,15 @@ def test_generate_description_historic_imagery(metadata: Tuple[CollectionMetadat
     metadata_auck, _ = metadata
     metadata_auck["category"] = "scanned-aerial-photos"
     metadata_auck["historic_survey_number"] = "SNC8844"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     description = "Scanned aerial imagery within the Auckland region captured in 2023."
     assert collection.stac["description"] == description
 
 
 def test_generate_description_event(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     _, metadata_hb = metadata
     metadata_hb["event_name"] = "Cyclone Gabrielle"
-    collection = ImageryCollection(metadata_hb)
+    collection = ImageryCollection(metadata_hb, any_epoch_datetime)
     description = "Orthophotography within the Hawke's Bay region captured in the 2023 flying season, \
 published as a record of the Cyclone Gabrielle event."
     assert collection.stac["description"] == description
diff --git a/scripts/stac/imagery/tests/generate_title_test.py b/scripts/stac/imagery/tests/generate_title_test.py
@@ -5,6 +5,7 @@
 
 from scripts.stac.imagery.collection import ImageryCollection
 from scripts.stac.imagery.metadata_constants import CollectionMetadata, MissingMetadataError
+from scripts.tests.datetimes_test import any_epoch_datetime
 
 
 # pylint: disable=duplicate-code
@@ -38,30 +39,30 @@ def setup() -> Generator[Tuple[CollectionMetadata, CollectionMetadata], None, No
 def test_generate_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     title = "Auckland 0.3m Rural Aerial Photos (2023)"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     assert collection.stac["title"] == title
 
 
 def test_generate_dem_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["category"] = "dem"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland LiDAR 0.3m DEM (2023)"
     assert collection.stac["title"] == title
 
 
 def test_generate_dsm_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["category"] = "dsm"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland LiDAR 0.3m DSM (2023)"
     assert collection.stac["title"] == title
 
 
 def test_generate_satellite_imagery_title(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["category"] = "satellite-imagery"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland 0.3m Satellite Imagery (2023)"
     assert collection.stac["title"] == title
 
@@ -71,7 +72,7 @@ def test_generate_historic_imagery_title(metadata: Tuple[CollectionMetadata, Col
     metadata_auck, _ = metadata
     metadata_auck["category"] = "scanned-aerial-photos"
     metadata_auck["historic_survey_number"] = "SNC8844"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     assert collection.stac["title"] == title
 
 
@@ -80,23 +81,23 @@ def test_generate_historic_imagery_title_missing_number(metadata: Tuple[Collecti
     metadata_auck["category"] = "scanned-aerial-photos"
     metadata_auck["historic_survey_number"] = None
     with pytest.raises(MissingMetadataError) as excinfo:
-        ImageryCollection(metadata_auck)
+        ImageryCollection(metadata_auck, any_epoch_datetime)
 
     assert "historic_survey_number" in str(excinfo.value)
 
 
 def test_generate_title_long_date(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["end_datetime"] = datetime(2024, 1, 1)
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland 0.3m Rural Aerial Photos (2023-2024)"
     assert collection.stac["title"] == title
 
 
 def test_generate_title_geographic_description(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     metadata_auck, _ = metadata
     metadata_auck["geographic_description"] = "Ponsonby"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Ponsonby 0.3m Rural Aerial Photos (2023)"
     assert collection.stac["title"] == title
 
@@ -105,7 +106,7 @@ def test_generate_title_event_imagery(metadata: Tuple[CollectionMetadata, Collec
     _, metadata_hb = metadata
     metadata_hb["geographic_description"] = "Hawke's Bay Cyclone Gabrielle"
     metadata_hb["event_name"] = "Cyclone Gabrielle"
-    collection = ImageryCollection(metadata_hb)
+    collection = ImageryCollection(metadata_hb, any_epoch_datetime)
     title = "Hawke's Bay Cyclone Gabrielle 0.3m Rural Aerial Photos (2023)"
     assert collection.stac["title"] == title
 
@@ -115,7 +116,7 @@ def test_generate_title_event_elevation(metadata: Tuple[CollectionMetadata, Coll
     metadata_hb["category"] = "dsm"
     metadata_hb["geographic_description"] = "Hawke's Bay Cyclone Gabrielle"
     metadata_hb["event_name"] = "Cyclone Gabrielle"
-    collection = ImageryCollection(metadata_hb)
+    collection = ImageryCollection(metadata_hb, any_epoch_datetime)
     title = "Hawke's Bay - Hawke's Bay Cyclone Gabrielle LiDAR 0.3m DSM (2023)"
     assert collection.stac["title"] == title
 
@@ -125,7 +126,7 @@ def test_generate_title_event_satellite_imagery(metadata: Tuple[CollectionMetada
     metadata_hb["category"] = "satellite-imagery"
     metadata_hb["geographic_description"] = "Hawke's Bay Cyclone Gabrielle"
     metadata_hb["event_name"] = "Cyclone Gabrielle"
-    collection = ImageryCollection(metadata_hb)
+    collection = ImageryCollection(metadata_hb, any_epoch_datetime)
     title = "Hawke's Bay Cyclone Gabrielle 0.3m Satellite Imagery (2023)"
     assert collection.stac["title"] == title
 
@@ -134,15 +135,15 @@ def test_generate_dsm_title_preview(metadata: Tuple[CollectionMetadata, Collecti
     metadata_auck, _ = metadata
     metadata_auck["category"] = "dsm"
     metadata_auck["lifecycle"] = "preview"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland LiDAR 0.3m DSM (2023) - Preview"
     assert collection.stac["title"] == title
 
 
 def test_generate_imagery_title_draft(metadata: Tuple[CollectionMetadata, CollectionMetadata]) -> None:
     _, metadata_hb = metadata
     metadata_hb["lifecycle"] = "ongoing"
-    collection = ImageryCollection(metadata_hb)
+    collection = ImageryCollection(metadata_hb, any_epoch_datetime)
     title = "Hawke's Bay 0.3m Rural Aerial Photos (2023) - Draft"
     assert collection.stac["title"] == title
 
@@ -151,7 +152,7 @@ def test_generate_imagery_title_empty_optional_str(metadata: Tuple[CollectionMet
     metadata_auck, _ = metadata
     metadata_auck["geographic_description"] = ""
     metadata_auck["event_name"] = ""
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland 0.3m Rural Aerial Photos (2023)"
     assert collection.stac["title"] == title
 
@@ -160,6 +161,6 @@ def test_generate_imagery_title_with_event(metadata: Tuple[CollectionMetadata, C
     metadata_auck, _ = metadata
     metadata_auck["geographic_description"] = "Auckland Forest Assessment"
     metadata_auck["event_name"] = "Forest Assessment"
-    collection = ImageryCollection(metadata_auck)
+    collection = ImageryCollection(metadata_auck, any_epoch_datetime)
     title = "Auckland Forest Assessment 0.3m Rural Aerial Photos (2023)"
     assert collection.stac["title"] == title