From ba30e04823d0dfb3a5edbb123a637147bd6d6daa Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Mon, 21 Aug 2023 11:12:00 +1200 Subject: [PATCH] WIP --- scripts/collection_from_items.py | 4 +- scripts/files/fs_s3.py | 7 ++- scripts/tests/data/input_dem_ascii.asc | 35 +++++++++++++++ scripts/translate_ascii.py | 59 ++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 scripts/tests/data/input_dem_ascii.asc create mode 100644 scripts/translate_ascii.py diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index 642697ad4..45b85a72c 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -7,7 +7,7 @@ from linz_logger import get_log from scripts.cli.cli_helper import coalesce_multi_single -from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri +from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_uri from scripts.logging.time_helper import time_in_ms from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.provider import Provider, ProviderRole @@ -54,7 +54,7 @@ def main() -> None: s3_client = client("s3") - files_to_read = list_json_in_uri(uri, s3_client) + files_to_read = list_uri(uri, s3_client, extension=".json") start_time = time_in_ms() for key, result in get_object_parallel_multithreading( diff --git a/scripts/files/fs_s3.py b/scripts/files/fs_s3.py index e1f4bf775..42221c129 100644 --- a/scripts/files/fs_s3.py +++ b/scripts/files/fs_s3.py @@ -7,7 +7,6 @@ from linz_logger import get_log from scripts.aws.aws_helper import get_session, parse_path -from scripts.files.files_helper import is_json from scripts.logging.time_helper import time_in_ms @@ -164,7 +163,7 @@ def prefix_from_path(path: str) -> str: return path.replace(f"s3://{bucket_name}/", "") -def list_json_in_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]: +def list_uri(uri: str, s3_client: Optional[boto3.client], extension: Optional[str] = None) -> List[str]: """Get the `JSON` files from a s3 path Args: @@ -183,8 +182,8 @@ def list_json_in_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]: for response in response_iterator: for contents_data in response["Contents"]: key = contents_data["Key"] - if not is_json(key): - get_log().trace("skipping file not json", file=key, action="collection_from_items", reason="skip") + if extension not in key: + get_log().trace("skipping file not specified extension", file=key, extension=extension, reason="skip") continue files.append(key) get_log().info("Files Listed", number_of_files=len(files)) diff --git a/scripts/tests/data/input_dem_ascii.asc b/scripts/tests/data/input_dem_ascii.asc new file mode 100644 index 000000000..cb1868a59 --- /dev/null +++ b/scripts/tests/data/input_dem_ascii.asc @@ -0,0 +1,35 @@ +ncols 19 +nrows 29 +xllcorner 1579360.000000000000 +yllcorner 5219515.000000000000 +cellsize 25.000000000000 +NODATA_value -9999 + 26.95999908447265625 28.059999465942382812 28.780000686645507812 27.909999847412109375 22.649999618530273438 22.979999542236328125 28.1100006103515625 28.350000381469726562 28.149999618530273438 28.090000152587890625 27.93000030517578125 27.760000228881835938 27.559999465942382812 27.3600006103515625 27.049999237060546875 26.93000030517578125 26.68000030517578125 26.45999908447265625 26.3899993896484375 + 28.809999465942382812 28.68000030517578125 26.70999908447265625 28.329999923706054688 22.350000381469726562 22.25 26.520000457763671875 28.270000457763671875 28.159999847412109375 28.020000457763671875 27.93000030517578125 27.70999908447265625 27.479999542236328125 27.29000091552734375 27.149999618530273438 26.920000076293945312 26.6100006103515625 26.45999908447265625 26.350000381469726562 + 28.8600006103515625 28.770000457763671875 28.340000152587890625 22.729999542236328125 22.090000152587890625 22.450000762939453125 25.54000091552734375 28.030000686645507812 28.129999160766601562 27.870000839233398438 27.870000839233398438 27.68000030517578125 27.510000228881835938 27.299999237060546875 27.329999923706054688 26.93000030517578125 26.690000534057617188 26.479999542236328125 26.299999237060546875 + 28.829999923706054688 28.729999542236328125 28.56999969482421875 23.340000152587890625 24.93000030517578125 22.059999465942382812 21.3899993896484375 26.809999465942382812 27.969999313354492188 27.829999923706054688 27.729999542236328125 27.700000762939453125 27.510000228881835938 27.229999542236328125 27.1100006103515625 26.8899993896484375 26.659999847412109375 26.489999771118164062 26.20999908447265625 + 28.739999771118164062 28.56999969482421875 28.549999237060546875 28.399999618530273438 28.29000091552734375 24.31999969482421875 21.590000152587890625 22.149999618530273438 25.020000457763671875 27.770000457763671875 27.81999969482421875 27.6399993896484375 27.489999771118164062 27.239999771118164062 27.020000457763671875 26.969999313354492188 26.6100006103515625 26.479999542236328125 26.3899993896484375 + 28.70999908447265625 28.56999969482421875 28.3899993896484375 28.3899993896484375 28.219999313354492188 28.030000686645507812 22.93000030517578125 20.8899993896484375 21.450000762939453125 22.3600006103515625 27.549999237060546875 27.600000381469726562 27.399999618530273438 27.090000152587890625 27.059999465942382812 26.8899993896484375 26.739999771118164062 26.629999160766601562 26.3899993896484375 + 28.6399993896484375 28.590000152587890625 28.3899993896484375 28.190000534057617188 28.219999313354492188 28.010000228881835938 21.45999908447265625 21.229999542236328125 21.1399993896484375 20.700000762939453125 20.479999542236328125 27.549999237060546875 24.229999542236328125 26.340000152587890625 27.020000457763671875 26.81999969482421875 26.729999542236328125 26.739999771118164062 26.379999160766601562 + 28.620000839233398438 28.379999160766601562 28.340000152587890625 28.190000534057617188 28.059999465942382812 26.8600006103515625 21.090000152587890625 23.729999542236328125 20.850000381469726562 21.030000686645507812 20.549999237060546875 19.909999847412109375 20.149999618530273438 19.75 20.969999313354492188 19.340000152587890625 19.56999969482421875 24.690000534057617188 26.379999160766601562 + 28.549999237060546875 28.43000030517578125 28.379999160766601562 28.200000762939453125 28.079999923706054688 28 23.159999847412109375 21.530000686645507812 25.159999847412109375 23.20999908447265625 20.31999969482421875 20.270000457763671875 19.700000762939453125 19.450000762939453125 19.520000457763671875 19.120000839233398438 21.68000030517578125 23.049999237060546875 26.45999908447265625 + 28.549999237060546875 28.3600006103515625 28.229999542236328125 28.25 28.04000091552734375 27.950000762939453125 27.799999237060546875 27.700000762939453125 27.450000762939453125 27.299999237060546875 20.909999847412109375 19.6100006103515625 20.010000228881835938 19.409999847412109375 19.120000839233398438 18.770000457763671875 18.25 27.020000457763671875 26.379999160766601562 + 28.6399993896484375 28.5 28.370000839233398438 28.370000839233398438 28.129999160766601562 28.010000228881835938 27.739999771118164062 27.489999771118164062 27.31999969482421875 27.25 26.299999237060546875 20.270000457763671875 19.8600006103515625 19.25 18.760000228881835938 25.739999771118164062 18.6399993896484375 21.29000091552734375 26.68000030517578125 + 28.649999618530273438 28.719999313354492188 28.649999618530273438 28.3899993896484375 28.159999847412109375 27.950000762939453125 27.81999969482421875 27.530000686645507812 27.440000534057617188 27.149999618530273438 27.079999923706054688 26.879999160766601562 21.219999313354492188 18.729999542236328125 19.030000686645507812 20.620000839233398438 18.6399993896484375 18.659999847412109375 26.45999908447265625 + 28.840000152587890625 28.8600006103515625 28.68000030517578125 28.379999160766601562 28.129999160766601562 27.899999618530273438 27.8600006103515625 27.629999160766601562 27.370000839233398438 27.229999542236328125 26.909999847412109375 26.93000030517578125 26.950000762939453125 23.260000228881835938 18.6100006103515625 18.270000457763671875 18.440000534057617188 18.06999969482421875 26.719999313354492188 + 29.100000381469726562 29 28.579999923706054688 28.280000686645507812 28.100000381469726562 27.8899993896484375 27.760000228881835938 27.629999160766601562 27.440000534057617188 27.1399993896484375 27.04000091552734375 26.780000686645507812 26.739999771118164062 26.620000839233398438 18.75 18.020000457763671875 17.920000076293945312 17.70999908447265625 26.95999908447265625 + 29.299999237060546875 28.979999542236328125 28.6399993896484375 28.379999160766601562 28.20999908447265625 27.979999542236328125 27.809999465942382812 27.690000534057617188 27.620000839233398438 27.31999969482421875 26.95999908447265625 26.969999313354492188 23.670000076293945312 22.870000839233398438 19.1100006103515625 18.059999465942382812 16.95999908447265625 18.729999542236328125 24.700000762939453125 + 29.340000152587890625 29.090000152587890625 28.809999465942382812 28.549999237060546875 28.229999542236328125 27.969999313354492188 27.840000152587890625 27.440000534057617188 27.43000030517578125 27.049999237060546875 27.090000152587890625 26.309999465942382812 23.350000381469726562 20.81999969482421875 17.620000839233398438 17.68000030517578125 18.409999847412109375 17.469999313354492188 17.68000030517578125 + 29.469999313354492188 29.25 28.979999542236328125 28.530000686645507812 28.079999923706054688 27.8600006103515625 27.670000076293945312 27.3899993896484375 27.1100006103515625 27.010000228881835938 24.829999923706054688 17.8600006103515625 16.93000030517578125 16.879999160766601562 17.329999923706054688 17.219999313354492188 17.620000839233398438 16.909999847412109375 25.6399993896484375 + 29.450000762939453125 29.229999542236328125 28.920000076293945312 28.510000228881835938 28.049999237060546875 27.739999771118164062 27.489999771118164062 27.29000091552734375 27.1399993896484375 26.75 25.1399993896484375 16.5 16.190000534057617188 16.420000076293945312 18.799999237060546875 17.190000534057617188 17.629999160766601562 17.059999465942382812 26.1100006103515625 + 29.260000228881835938 29.010000228881835938 28.739999771118164062 28.399999618530273438 28.1100006103515625 27.6399993896484375 27.520000457763671875 27.270000457763671875 26.95999908447265625 26.559999465942382812 16.70999908447265625 16.329999923706054688 16.280000686645507812 16.8600006103515625 18.559999465942382812 17.940000534057617188 16.54000091552734375 16.590000152587890625 25.840000152587890625 + 29 28.770000457763671875 28.530000686645507812 28.170000076293945312 28.159999847412109375 27.909999847412109375 27.420000076293945312 27.170000076293945312 26.879999160766601562 26.469999313354492188 17.479999542236328125 16.45999908447265625 15.81999969482421875 16.45999908447265625 16.280000686645507812 15.56999969482421875 16.479999542236328125 16.600000381469726562 25.729999542236328125 + 28.8600006103515625 28.530000686645507812 28.43000030517578125 28.25 27.81999969482421875 27.75 27.409999847412109375 27.04000091552734375 26.829999923706054688 26.56999969482421875 26.239999771118164062 21.899999618530273438 16.309999465942382812 15.760000228881835938 16.129999160766601562 16.549999237060546875 15.710000038146972656 20.579999923706054688 25.56999969482421875 + 28.729999542236328125 28.549999237060546875 28.010000228881835938 27.850000381469726562 27.68000030517578125 27.600000381469726562 27.3899993896484375 26.969999313354492188 26.79000091552734375 26.590000152587890625 26.260000228881835938 25.95999908447265625 18.280000686645507812 15.420000076293945312 16.1399993896484375 15.159999847412109375 15.56999969482421875 17.479999542236328125 25.600000381469726562 + 28.440000534057617188 28.239999771118164062 27.8899993896484375 27.620000839233398438 27.56999969482421875 27.43000030517578125 27.1100006103515625 26.879999160766601562 26.739999771118164062 26.489999771118164062 26.219999313354492188 26.020000457763671875 17.25 16.010000228881835938 16.030000686645507812 15.729999542236328125 15.5 15.310000419616699219 17.809999465942382812 + 28.280000686645507812 28.010000228881835938 27.719999313354492188 27.520000457763671875 27.299999237060546875 27.129999160766601562 26.969999313354492188 27.059999465942382812 26.620000839233398438 26.489999771118164062 26.399999618530273438 24.659999847412109375 15.090000152587890625 14.909999847412109375 14.760000228881835938 14.760000228881835938 15.449999809265136719 14.949999809265136719 15.850000381469726562 + 28.260000228881835938 27.95999908447265625 27.54000091552734375 27.219999313354492188 26.979999542236328125 26.8899993896484375 26.79000091552734375 26.70999908447265625 26.420000076293945312 26.340000152587890625 26.29000091552734375 24.100000381469726562 15.659999847412109375 21.309999465942382812 19.239999771118164062 16.299999237060546875 14.789999961853027344 14.659999847412109375 14.710000038146972656 + 28.200000762939453125 27.8600006103515625 27.479999542236328125 27.170000076293945312 26.8600006103515625 26.700000762939453125 26.56999969482421875 26.370000839233398438 26.379999160766601562 26.1100006103515625 26.020000457763671875 25.940000534057617188 25.6399993896484375 25.510000228881835938 25.280000686645507812 19.1399993896484375 14.859999656677246094 14.260000228881835938 13.829999923706054688 + 28.030000686645507812 27.729999542236328125 27.530000686645507812 27.149999618530273438 26.760000228881835938 26.440000534057617188 26.3600006103515625 26.219999313354492188 26.399999618530273438 26 25.8600006103515625 25.719999313354492188 25.579999923706054688 25.440000534057617188 25.190000534057617188 24.3600006103515625 16.25 14.060000419616699219 14.720000267028808594 + 27.8600006103515625 27.719999313354492188 27.379999160766601562 27.010000228881835938 26.549999237060546875 26.25 26.059999465942382812 26.06999969482421875 26.030000686645507812 25.840000152587890625 25.68000030517578125 25.510000228881835938 25.3899993896484375 25.260000228881835938 25.1399993896484375 25.120000839233398438 24.799999237060546875 16.909999847412109375 14.189999580383300781 + 27.489999771118164062 27.280000686645507812 27.100000381469726562 26.950000762939453125 26.54000091552734375 26.06999969482421875 25.979999542236328125 25.75 25.559999465942382812 25.659999847412109375 25.469999313354492188 25.370000839233398438 25.270000457763671875 25.1100006103515625 24.95999908447265625 25.159999847412109375 25.260000228881835938 24.780000686645507812 17.409999847412109375 diff --git a/scripts/translate_ascii.py b/scripts/translate_ascii.py new file mode 100644 index 000000000..1e929e0eb --- /dev/null +++ b/scripts/translate_ascii.py @@ -0,0 +1,59 @@ +import argparse +import os +import tempfile + +from scripts.aws.aws_helper import get_session, is_s3 +from scripts.files.fs import find_sidecars, write_all +from scripts.files.fs_s3 import bucket_name_from_path, list_uri +from scripts.gdal.gdal_helper import run_gdal + + +def main() -> None: + """Translates ascii files in a given path to tiffs and writes to target, + + Arguments: + --source - local or s3 path to source data + --target - local or s3 path to write converted tiffs + + examples: + python translate_ascii.py --source s3://linz-elevation-staging/test/sample-tests/ --target /tmp/ + python translate_ascii.py --source ./tests/data/ --target /tmp/ + """ + + parser = argparse.ArgumentParser() + parser.add_argument("--source", dest="source", required=True, help="S3 path to the input ascii files") + parser.add_argument("--target", dest="target", required=True, help="Output location path") + arguments = parser.parse_args() + + source = arguments.source + + with tempfile.TemporaryDirectory() as tmp_path: + if is_s3(source): + # list ascii files from s3 + s3 = get_session(source).client("s3") + asc_files = list_uri(source, s3, extension=".asc") + # reformat to full s3 path + asc_files = [f"s3://{bucket_name_from_path(source)}/{file}" for file in asc_files] + # download ascii files + asc_files = write_all(inputs=asc_files, target=tmp_path) + + else: + # list ascii files from local + asc_files = [os.path.join(source, file) for file in os.listdir(source) if file.endswith(".asc")] + + tiffs = [] + for asc in asc_files: + # translate from ascii to geotiff using GDAL + tiff = os.path.join(tmp_path, f"{asc.split('/')[-1].split('.')[0]}.tiff") + command = ["gdal_translate", "-of", "GTiff", asc, tiff] + run_gdal(command) + tiffs.append(tiff) + write_all(inputs=tiffs, target=arguments.target) + + # copy any sidecar files to target + sidecars = find_sidecars(asc_files, [".prj", ".tfw"]) + write_all(inputs=sidecars, target=arguments.target) + + +if __name__ == "__main__": + main()