From 5b240e7c97c27a3e6e9f59982c481e9d398e1d55 Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Mon, 6 May 2024 11:02:11 +1200 Subject: [PATCH] feat: add option to create footprints Co-Authored-By: Victor Engmark Co-Authored-By: Alice Fage Co-Authored-By: Megan Davidson --- .github/workflows/format-tests.yml | 12 ++++----- README.md | 2 +- scripts/standardise_validate.py | 16 +++++++++++ scripts/standardising.py | 43 +++++++++++++++++------------- 4 files changed, 47 insertions(+), 26 deletions(-) diff --git a/.github/workflows/format-tests.yml b/.github/workflows/format-tests.yml index d8d287857..0beefb408 100644 --- a/.github/workflows/format-tests.yml +++ b/.github/workflows/format-tests.yml @@ -32,23 +32,23 @@ jobs: - name: End to end test - Aerial Imagery run: | - docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m --create-footprints=true cmp --silent "${{ runner.temp }}/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829.tiff - name: End to end test - Elevation run: | - docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 30m + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 30m --create-footprints=true cmp --silent "${{ runner.temp }}/BK39_10000_0102.tiff" ./scripts/tests/data/output/BK39_10000_0102.tiff cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101.tiff - name: End to end test - Historical Aerial Imagery run: | - docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60m + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60m --create-footprints=true cmp --silent "${{ runner.temp }}/BQ31_5000_0608.tiff" ./scripts/tests/data/output/BQ31_5000_0608.tiff - name: End to end test - Cutline (Aerial Imagery) run: | - docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb --gsd 10m + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb --gsd 10m --create-footprints=true cmp --silent "${{ runner.temp }}/cutline/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829_cut.tiff - name: End to end test - Thumbnails (Topo50/Topo250) @@ -59,7 +59,7 @@ jobs: - name: End to end test - Restandardise Aerial Imagery run: | - docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/restandardise.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/restandardise/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/restandardise.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/restandardise/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m --create-footprints=true cmp --silent "${{ runner.temp }}/restandardise/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829.tiff - name: End to end test - Translate Ascii Files (Elevation) @@ -69,7 +69,7 @@ jobs: - name: End to end test - Remove empty files run: | - docker run -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01 --gsd 60m + docker run -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01 --gsd 60m --create-footprints=true empty_target_directory="$(find "${{ runner.temp }}/tmp-empty" -maxdepth 0 -type d -empty)" [[ -n "$empty_target_directory" ]] diff --git a/README.md b/README.md index d8195b5f5..ee676671b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Run `docker run topo-imagery python standardise_validate.py --help` to get the l - Example of local execution. This example uses the test data available on this repo and create the output will be created in a `~/tmp/` on the local machine (volume share with `Docker`): ```bash -docker run -v ${HOME}/tmp/:/tmp/:rw topo-imagery python standardise_validate.py --preset webp --from-file ./tests/data/aerial.json --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --target /tmp/ --source-epsg 2193 --target-epsg 2193 --gsd 10m +docker run -v ${HOME}/tmp/:/tmp/:rw topo-imagery python standardise_validate.py --preset webp --from-file ./tests/data/aerial.json --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --target /tmp/ --source-epsg 2193 --target-epsg 2193 --gsd 10m --create-footprints=true ``` To use an AWS test dataset (input located in an AWS S3 bucket), log into the AWS account and add the following arguments to the `docker run` command: diff --git a/scripts/standardise_validate.py b/scripts/standardise_validate.py index cd0cee9fe..07f20dab0 100644 --- a/scripts/standardise_validate.py +++ b/scripts/standardise_validate.py @@ -15,6 +15,14 @@ from scripts.standardising import run_standardising +def str_to_bool(value: str) -> bool: + if value == "true": + return True + if value == "false": + return False + raise argparse.ArgumentTypeError(f"Invalid boolean (must be exactly 'true' or 'false'): {value}") + + def main() -> None: # pylint: disable-msg=too-many-locals parser = argparse.ArgumentParser() @@ -30,6 +38,13 @@ def main() -> None: help="The target EPSG code. If different to source the imagery will be reprojected", ) parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True) + parser.add_argument( + "--create-footprints", + dest="create_footprints", + help="Create footprints for each tile ('true' / 'false')", + type=str_to_bool, + required=True, + ) parser.add_argument("--cutline", dest="cutline", help="Optional cutline to cut imagery to", required=False, nargs="?") parser.add_argument("--collection-id", dest="collection_id", help="Unique id for collection", required=True) parser.add_argument( @@ -60,6 +75,7 @@ def main() -> None: arguments.source_epsg, arguments.target_epsg, arguments.gsd, + arguments.create_footprints, arguments.target, ) diff --git a/scripts/standardising.py b/scripts/standardising.py index 0123b5a65..cfb170609 100644 --- a/scripts/standardising.py +++ b/scripts/standardising.py @@ -34,6 +34,7 @@ def run_standardising( source_epsg: str, target_epsg: str, gsd: str, + create_footprints: bool, target_output: str = "/tmp/", ) -> List[FileTiff]: """Run `standardising()` in parallel (`concurrency`). @@ -68,6 +69,7 @@ def run_standardising( target_epsg=target_epsg, target_output=target_output, gsd=gsd, + create_footprints=create_footprints, cutline=cutline, ), todo, @@ -107,6 +109,7 @@ def standardising( source_epsg: str, target_epsg: str, gsd: str, + create_footprints: bool, cutline: Optional[str], target_output: str = "/tmp/", ) -> Optional[FileTiff]: @@ -212,25 +215,27 @@ def standardising( with TiffFile(standardized_working_path) as file_handle: if any(tile_byte_count != 0 for tile_byte_count in file_handle.pages.first.tags["TileByteCounts"].value): - # Create footprint GeoJSON - run_gdal( - [ - "gdal_footprint", - "-t_srs", - EpsgCode.EPSG_4326, - "-max_points", - "unlimited", - "-simplify", - str(get_buffer_distance(gsd_to_float(gsd))), - ], - standardized_working_path, - footprint_tmp_path, - ) - write( - footprint_file_path, - read(footprint_tmp_path), - content_type=ContentType.GEOJSON.value, - ) + if create_footprints: + # Create footprint GeoJSON + run_gdal( + [ + "gdal_footprint", + "-t_srs", + EpsgCode.EPSG_4326, + "-max_points", + "unlimited", + "-simplify", + str(get_buffer_distance(gsd_to_float(gsd))), + ], + standardized_working_path, + footprint_tmp_path, + ) + write( + footprint_file_path, + read(footprint_tmp_path), + content_type=ContentType.GEOJSON.value, + ) + write(standardized_file_path, read(standardized_working_path), content_type=ContentType.GEOTIFF.value) return tiff