From 37ee07f413f93ab831e8bec4b656d29d31d78dbc Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Tue, 4 Jun 2024 03:20:39 +0000 Subject: [PATCH] feat: Set created/updated item properties TDE-1147 (#980) #### Motivation When creating datasets, we want to set these to the current datetime. #### Modification Injects the function to get the current datetime, to allow testing the exact values. #### Checklist - [x] Tests updated - [ ] Docs updated (N/A) - [x] Issue linked in Title --- scripts/datetimes.py | 16 +++++++++ scripts/stac/imagery/create_stac.py | 3 +- scripts/stac/imagery/item.py | 7 ++-- scripts/stac/imagery/tests/collection_test.py | 36 +++++++++++++------ scripts/stac/imagery/tests/item_test.py | 5 +-- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/scripts/datetimes.py b/scripts/datetimes.py index f3f8a30b4..df80709c2 100644 --- a/scripts/datetimes.py +++ b/scripts/datetimes.py @@ -39,6 +39,22 @@ def format_rfc_3339_nz_midnight_datetime_string(datetime_object: datetime) -> st return format_rfc_3339_datetime_string(datetime_utc) +def utc_now() -> datetime: + """ + Get the current datetime with UTC time zone + + Should return something close to the current time: + >>> current_timestamp = datetime.now().timestamp() + >>> current_timestamp - 5 < utc_now().timestamp() < current_timestamp + 5 + True + + Should have UTC time zone: + >>> utc_now().tzname() + 'UTC' + """ + return datetime.now(tz=timezone.utc) + + class NaiveDatetimeError(Exception): def __init__(self) -> None: super().__init__("Can't convert naive datetime to UTC") diff --git a/scripts/stac/imagery/create_stac.py b/scripts/stac/imagery/create_stac.py index 1acb232a2..fa71148cf 100644 --- a/scripts/stac/imagery/create_stac.py +++ b/scripts/stac/imagery/create_stac.py @@ -2,6 +2,7 @@ from linz_logger import get_log +from scripts.datetimes import utc_now from scripts.files.files_helper import get_file_name_from_path from scripts.files.geotiff import get_extents from scripts.gdal.gdal_helper import gdal_info @@ -35,7 +36,7 @@ def create_item( geometry, bbox = get_extents(gdalinfo_result) - item = ImageryItem(id_, file) + item = ImageryItem(id_, file, utc_now) item.update_datetime(start_datetime, end_datetime) item.update_spatial(geometry, bbox) item.add_collection(collection_id) diff --git a/scripts/stac/imagery/item.py b/scripts/stac/imagery/item.py index 651f6b553..1efae2716 100644 --- a/scripts/stac/imagery/item.py +++ b/scripts/stac/imagery/item.py @@ -1,5 +1,6 @@ import os -from typing import Any, Dict, Tuple +from datetime import datetime +from typing import Any, Callable, Dict, Tuple from scripts.datetimes import format_rfc_3339_datetime_string from scripts.files import fs @@ -12,9 +13,10 @@ class ImageryItem: stac: Dict[str, Any] - def __init__(self, id_: str, file: str) -> None: + def __init__(self, id_: str, file: str, now: Callable[[], datetime]) -> None: file_content = fs.read(file) file_modified_datetime = format_rfc_3339_datetime_string(modified(file)) + now_string = format_rfc_3339_datetime_string(now()) self.stac = { "type": "Feature", "stac_version": STAC_VERSION, @@ -32,6 +34,7 @@ def __init__(self, id_: str, file: str) -> None: } }, "stac_extensions": [StacExtensions.file.value], + "properties": {"created": now_string, "updated": now_string}, } def update_datetime(self, start_datetime: str, end_datetime: str) -> None: diff --git a/scripts/stac/imagery/tests/collection_test.py b/scripts/stac/imagery/tests/collection_test.py index 9ad0d8125..ef12b5720 100644 --- a/scripts/stac/imagery/tests/collection_test.py +++ b/scripts/stac/imagery/tests/collection_test.py @@ -4,12 +4,13 @@ from datetime import datetime, timezone from shutil import rmtree from tempfile import mkdtemp -from typing import Generator +from typing import Callable, Generator import pytest import shapely.geometry from pytest_subtests import SubTests +from scripts.datetimes import format_rfc_3339_datetime_string from scripts.files.fs import read from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.item import ImageryItem @@ -113,12 +114,20 @@ def test_interval_updated_from_existing(metadata: CollectionMetadata) -> None: assert collection.stac["extent"]["temporal"]["interval"] == [["2021-01-27T00:00:00Z", "2021-02-20T00:00:00Z"]] +def fixed_now_function(now: datetime) -> Callable[[], datetime]: + def func() -> datetime: + return now + + return func + + def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: collection = ImageryCollection(metadata) item_file_path = "./scripts/tests/data/empty.tiff" modified_datetime = datetime(2001, 2, 3, hour=4, minute=5, second=6, tzinfo=timezone.utc) os.utime(item_file_path, times=(any_epoch_datetime().timestamp(), modified_datetime.timestamp())) - item = ImageryItem("BR34_5000_0304", item_file_path) + now = any_epoch_datetime() + item = ImageryItem("BR34_5000_0304", item_file_path, fixed_now_function(now)) geometry = { "type": "Polygon", "coordinates": [[1799667.5, 5815977.0], [1800422.5, 5815977.0], [1800422.5, 5814986.0], [1799667.5, 5814986.0]], @@ -131,13 +140,17 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: collection.add_item(item.stac) - with subtests.test(): - assert { - "file:checksum": "122097b5d2b049c6ffdf608af28c4ba2744fad7f03046d1f58b2523402f30577f618", - "rel": "item", - "href": "./BR34_5000_0304.json", - "type": "application/json", - } in collection.stac["links"] + links = collection.stac["links"].copy() + + with subtests.test(msg="File checksum heuristic"): + # The checksum changes based on the contents + assert links[1].pop("file:checksum").startswith("1220") + + with subtests.test(msg="Main links content"): + assert [ + {"href": "./collection.json", "rel": "self", "type": "application/json"}, + {"rel": "item", "href": "./BR34_5000_0304.json", "type": "application/json"}, + ] == links with subtests.test(): assert collection.stac["extent"]["temporal"]["interval"] == [[start_datetime, end_datetime]] @@ -146,7 +159,10 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] for property_name in ["created", "updated"]: - with subtests.test(msg=f"{property_name} property"): + with subtests.test(msg=f"item properties.{property_name}"): + assert item.stac["properties"][property_name] == format_rfc_3339_datetime_string(now) + + with subtests.test(msg=f"item assets.visual.{property_name}"): assert item.stac["assets"]["visual"][property_name] == "2001-02-03T04:05:06Z" diff --git a/scripts/stac/imagery/tests/item_test.py b/scripts/stac/imagery/tests/item_test.py index 4b3be4858..e180ccd4b 100644 --- a/scripts/stac/imagery/tests/item_test.py +++ b/scripts/stac/imagery/tests/item_test.py @@ -7,6 +7,7 @@ from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.item import ImageryItem from scripts.stac.imagery.metadata_constants import CollectionMetadata +from scripts.tests.datetimes_test import any_epoch_datetime def test_imagery_stac_item(mocker: MockerFixture, subtests: SubTests) -> None: @@ -23,7 +24,7 @@ def test_imagery_stac_item(mocker: MockerFixture, subtests: SubTests) -> None: start_datetime = "2021-01-27T00:00:00Z" end_datetime = "2021-01-27T00:00:00Z" - item = ImageryItem(id_, path) + item = ImageryItem(id_, path, any_epoch_datetime) item.update_spatial(geometry, bbox) item.update_datetime(start_datetime, end_datetime) # checks @@ -77,7 +78,7 @@ def test_imagery_add_collection(mocker: MockerFixture, subtests: SubTests) -> No path = "./scripts/tests/data/empty.tiff" id_ = get_file_name_from_path(path) mocker.patch("scripts.files.fs.read", return_value=b"") - item = ImageryItem(id_, path) + item = ImageryItem(id_, path, any_epoch_datetime) item.add_collection(collection.stac["id"])