WIP

linz · Aug 20, 2023 · ba30e04 · ba30e04
1 parent 8ab269d
commit ba30e04
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 6 deletions.
diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py
@@ -7,7 +7,7 @@
 from linz_logger import get_log
 
 from scripts.cli.cli_helper import coalesce_multi_single
-from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri
+from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_uri
 from scripts.logging.time_helper import time_in_ms
 from scripts.stac.imagery.collection import ImageryCollection
 from scripts.stac.imagery.provider import Provider, ProviderRole
@@ -54,7 +54,7 @@ def main() -> None:
 
     s3_client = client("s3")
 
-    files_to_read = list_json_in_uri(uri, s3_client)
+    files_to_read = list_uri(uri, s3_client, extension=".json")
 
     start_time = time_in_ms()
     for key, result in get_object_parallel_multithreading(

diff --git a/scripts/files/fs_s3.py b/scripts/files/fs_s3.py
@@ -7,7 +7,6 @@
 from linz_logger import get_log
 
 from scripts.aws.aws_helper import get_session, parse_path
-from scripts.files.files_helper import is_json
 from scripts.logging.time_helper import time_in_ms
 
 
@@ -164,7 +163,7 @@ def prefix_from_path(path: str) -> str:
     return path.replace(f"s3://{bucket_name}/", "")
 
 
-def list_json_in_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]:
+def list_uri(uri: str, s3_client: Optional[boto3.client], extension: Optional[str] = None) -> List[str]:
     """Get the `JSON` files from a s3 path
 
     Args:
@@ -183,8 +182,8 @@ def list_json_in_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]:
     for response in response_iterator:
         for contents_data in response["Contents"]:
             key = contents_data["Key"]
-            if not is_json(key):
-                get_log().trace("skipping file not json", file=key, action="collection_from_items", reason="skip")
+            if extension not in key:
+                get_log().trace("skipping file not specified extension", file=key, extension=extension, reason="skip")
                 continue
             files.append(key)
     get_log().info("Files Listed", number_of_files=len(files))

diff --git a/scripts/tests/data/input_dem_ascii.asc b/scripts/tests/data/input_dem_ascii.asc
@@ -0,0 +1,35 @@
+ncols        19
+nrows        29
+xllcorner    1579360.000000000000
+yllcorner    5219515.000000000000
+cellsize     25.000000000000
+NODATA_value  -9999
+ 26.95999908447265625 28.059999465942382812 28.780000686645507812 27.909999847412109375 22.649999618530273438 22.979999542236328125 28.1100006103515625 28.350000381469726562 28.149999618530273438 28.090000152587890625 27.93000030517578125 27.760000228881835938 27.559999465942382812 27.3600006103515625 27.049999237060546875 26.93000030517578125 26.68000030517578125 26.45999908447265625 26.3899993896484375
+ 28.809999465942382812 28.68000030517578125 26.70999908447265625 28.329999923706054688 22.350000381469726562 22.25 26.520000457763671875 28.270000457763671875 28.159999847412109375 28.020000457763671875 27.93000030517578125 27.70999908447265625 27.479999542236328125 27.29000091552734375 27.149999618530273438 26.920000076293945312 26.6100006103515625 26.45999908447265625 26.350000381469726562
+ 28.8600006103515625 28.770000457763671875 28.340000152587890625 22.729999542236328125 22.090000152587890625 22.450000762939453125 25.54000091552734375 28.030000686645507812 28.129999160766601562 27.870000839233398438 27.870000839233398438 27.68000030517578125 27.510000228881835938 27.299999237060546875 27.329999923706054688 26.93000030517578125 26.690000534057617188 26.479999542236328125 26.299999237060546875
+ 28.829999923706054688 28.729999542236328125 28.56999969482421875 23.340000152587890625 24.93000030517578125 22.059999465942382812 21.3899993896484375 26.809999465942382812 27.969999313354492188 27.829999923706054688 27.729999542236328125 27.700000762939453125 27.510000228881835938 27.229999542236328125 27.1100006103515625 26.8899993896484375 26.659999847412109375 26.489999771118164062 26.20999908447265625
+ 28.739999771118164062 28.56999969482421875 28.549999237060546875 28.399999618530273438 28.29000091552734375 24.31999969482421875 21.590000152587890625 22.149999618530273438 25.020000457763671875 27.770000457763671875 27.81999969482421875 27.6399993896484375 27.489999771118164062 27.239999771118164062 27.020000457763671875 26.969999313354492188 26.6100006103515625 26.479999542236328125 26.3899993896484375
+ 28.70999908447265625 28.56999969482421875 28.3899993896484375 28.3899993896484375 28.219999313354492188 28.030000686645507812 22.93000030517578125 20.8899993896484375 21.450000762939453125 22.3600006103515625 27.549999237060546875 27.600000381469726562 27.399999618530273438 27.090000152587890625 27.059999465942382812 26.8899993896484375 26.739999771118164062 26.629999160766601562 26.3899993896484375
+ 28.6399993896484375 28.590000152587890625 28.3899993896484375 28.190000534057617188 28.219999313354492188 28.010000228881835938 21.45999908447265625 21.229999542236328125 21.1399993896484375 20.700000762939453125 20.479999542236328125 27.549999237060546875 24.229999542236328125 26.340000152587890625 27.020000457763671875 26.81999969482421875 26.729999542236328125 26.739999771118164062 26.379999160766601562
+ 28.620000839233398438 28.379999160766601562 28.340000152587890625 28.190000534057617188 28.059999465942382812 26.8600006103515625 21.090000152587890625 23.729999542236328125 20.850000381469726562 21.030000686645507812 20.549999237060546875 19.909999847412109375 20.149999618530273438 19.75 20.969999313354492188 19.340000152587890625 19.56999969482421875 24.690000534057617188 26.379999160766601562
+ 28.549999237060546875 28.43000030517578125 28.379999160766601562 28.200000762939453125 28.079999923706054688 28 23.159999847412109375 21.530000686645507812 25.159999847412109375 23.20999908447265625 20.31999969482421875 20.270000457763671875 19.700000762939453125 19.450000762939453125 19.520000457763671875 19.120000839233398438 21.68000030517578125 23.049999237060546875 26.45999908447265625
+ 28.549999237060546875 28.3600006103515625 28.229999542236328125 28.25 28.04000091552734375 27.950000762939453125 27.799999237060546875 27.700000762939453125 27.450000762939453125 27.299999237060546875 20.909999847412109375 19.6100006103515625 20.010000228881835938 19.409999847412109375 19.120000839233398438 18.770000457763671875 18.25 27.020000457763671875 26.379999160766601562
+ 28.6399993896484375 28.5 28.370000839233398438 28.370000839233398438 28.129999160766601562 28.010000228881835938 27.739999771118164062 27.489999771118164062 27.31999969482421875 27.25 26.299999237060546875 20.270000457763671875 19.8600006103515625 19.25 18.760000228881835938 25.739999771118164062 18.6399993896484375 21.29000091552734375 26.68000030517578125
+ 28.649999618530273438 28.719999313354492188 28.649999618530273438 28.3899993896484375 28.159999847412109375 27.950000762939453125 27.81999969482421875 27.530000686645507812 27.440000534057617188 27.149999618530273438 27.079999923706054688 26.879999160766601562 21.219999313354492188 18.729999542236328125 19.030000686645507812 20.620000839233398438 18.6399993896484375 18.659999847412109375 26.45999908447265625
+ 28.840000152587890625 28.8600006103515625 28.68000030517578125 28.379999160766601562 28.129999160766601562 27.899999618530273438 27.8600006103515625 27.629999160766601562 27.370000839233398438 27.229999542236328125 26.909999847412109375 26.93000030517578125 26.950000762939453125 23.260000228881835938 18.6100006103515625 18.270000457763671875 18.440000534057617188 18.06999969482421875 26.719999313354492188
+ 29.100000381469726562 29 28.579999923706054688 28.280000686645507812 28.100000381469726562 27.8899993896484375 27.760000228881835938 27.629999160766601562 27.440000534057617188 27.1399993896484375 27.04000091552734375 26.780000686645507812 26.739999771118164062 26.620000839233398438 18.75 18.020000457763671875 17.920000076293945312 17.70999908447265625 26.95999908447265625
+ 29.299999237060546875 28.979999542236328125 28.6399993896484375 28.379999160766601562 28.20999908447265625 27.979999542236328125 27.809999465942382812 27.690000534057617188 27.620000839233398438 27.31999969482421875 26.95999908447265625 26.969999313354492188 23.670000076293945312 22.870000839233398438 19.1100006103515625 18.059999465942382812 16.95999908447265625 18.729999542236328125 24.700000762939453125
+ 29.340000152587890625 29.090000152587890625 28.809999465942382812 28.549999237060546875 28.229999542236328125 27.969999313354492188 27.840000152587890625 27.440000534057617188 27.43000030517578125 27.049999237060546875 27.090000152587890625 26.309999465942382812 23.350000381469726562 20.81999969482421875 17.620000839233398438 17.68000030517578125 18.409999847412109375 17.469999313354492188 17.68000030517578125
+ 29.469999313354492188 29.25 28.979999542236328125 28.530000686645507812 28.079999923706054688 27.8600006103515625 27.670000076293945312 27.3899993896484375 27.1100006103515625 27.010000228881835938 24.829999923706054688 17.8600006103515625 16.93000030517578125 16.879999160766601562 17.329999923706054688 17.219999313354492188 17.620000839233398438 16.909999847412109375 25.6399993896484375
+ 29.450000762939453125 29.229999542236328125 28.920000076293945312 28.510000228881835938 28.049999237060546875 27.739999771118164062 27.489999771118164062 27.29000091552734375 27.1399993896484375 26.75 25.1399993896484375 16.5 16.190000534057617188 16.420000076293945312 18.799999237060546875 17.190000534057617188 17.629999160766601562 17.059999465942382812 26.1100006103515625
+ 29.260000228881835938 29.010000228881835938 28.739999771118164062 28.399999618530273438 28.1100006103515625 27.6399993896484375 27.520000457763671875 27.270000457763671875 26.95999908447265625 26.559999465942382812 16.70999908447265625 16.329999923706054688 16.280000686645507812 16.8600006103515625 18.559999465942382812 17.940000534057617188 16.54000091552734375 16.590000152587890625 25.840000152587890625
+ 29 28.770000457763671875 28.530000686645507812 28.170000076293945312 28.159999847412109375 27.909999847412109375 27.420000076293945312 27.170000076293945312 26.879999160766601562 26.469999313354492188 17.479999542236328125 16.45999908447265625 15.81999969482421875 16.45999908447265625 16.280000686645507812 15.56999969482421875 16.479999542236328125 16.600000381469726562 25.729999542236328125
+ 28.8600006103515625 28.530000686645507812 28.43000030517578125 28.25 27.81999969482421875 27.75 27.409999847412109375 27.04000091552734375 26.829999923706054688 26.56999969482421875 26.239999771118164062 21.899999618530273438 16.309999465942382812 15.760000228881835938 16.129999160766601562 16.549999237060546875 15.710000038146972656 20.579999923706054688 25.56999969482421875
+ 28.729999542236328125 28.549999237060546875 28.010000228881835938 27.850000381469726562 27.68000030517578125 27.600000381469726562 27.3899993896484375 26.969999313354492188 26.79000091552734375 26.590000152587890625 26.260000228881835938 25.95999908447265625 18.280000686645507812 15.420000076293945312 16.1399993896484375 15.159999847412109375 15.56999969482421875 17.479999542236328125 25.600000381469726562
+ 28.440000534057617188 28.239999771118164062 27.8899993896484375 27.620000839233398438 27.56999969482421875 27.43000030517578125 27.1100006103515625 26.879999160766601562 26.739999771118164062 26.489999771118164062 26.219999313354492188 26.020000457763671875 17.25 16.010000228881835938 16.030000686645507812 15.729999542236328125 15.5 15.310000419616699219 17.809999465942382812
+ 28.280000686645507812 28.010000228881835938 27.719999313354492188 27.520000457763671875 27.299999237060546875 27.129999160766601562 26.969999313354492188 27.059999465942382812 26.620000839233398438 26.489999771118164062 26.399999618530273438 24.659999847412109375 15.090000152587890625 14.909999847412109375 14.760000228881835938 14.760000228881835938 15.449999809265136719 14.949999809265136719 15.850000381469726562
+ 28.260000228881835938 27.95999908447265625 27.54000091552734375 27.219999313354492188 26.979999542236328125 26.8899993896484375 26.79000091552734375 26.70999908447265625 26.420000076293945312 26.340000152587890625 26.29000091552734375 24.100000381469726562 15.659999847412109375 21.309999465942382812 19.239999771118164062 16.299999237060546875 14.789999961853027344 14.659999847412109375 14.710000038146972656
+ 28.200000762939453125 27.8600006103515625 27.479999542236328125 27.170000076293945312 26.8600006103515625 26.700000762939453125 26.56999969482421875 26.370000839233398438 26.379999160766601562 26.1100006103515625 26.020000457763671875 25.940000534057617188 25.6399993896484375 25.510000228881835938 25.280000686645507812 19.1399993896484375 14.859999656677246094 14.260000228881835938 13.829999923706054688
+ 28.030000686645507812 27.729999542236328125 27.530000686645507812 27.149999618530273438 26.760000228881835938 26.440000534057617188 26.3600006103515625 26.219999313354492188 26.399999618530273438 26 25.8600006103515625 25.719999313354492188 25.579999923706054688 25.440000534057617188 25.190000534057617188 24.3600006103515625 16.25 14.060000419616699219 14.720000267028808594
+ 27.8600006103515625 27.719999313354492188 27.379999160766601562 27.010000228881835938 26.549999237060546875 26.25 26.059999465942382812 26.06999969482421875 26.030000686645507812 25.840000152587890625 25.68000030517578125 25.510000228881835938 25.3899993896484375 25.260000228881835938 25.1399993896484375 25.120000839233398438 24.799999237060546875 16.909999847412109375 14.189999580383300781
+ 27.489999771118164062 27.280000686645507812 27.100000381469726562 26.950000762939453125 26.54000091552734375 26.06999969482421875 25.979999542236328125 25.75 25.559999465942382812 25.659999847412109375 25.469999313354492188 25.370000839233398438 25.270000457763671875 25.1100006103515625 24.95999908447265625 25.159999847412109375 25.260000228881835938 24.780000686645507812 17.409999847412109375
diff --git a/scripts/translate_ascii.py b/scripts/translate_ascii.py
@@ -0,0 +1,59 @@
+import argparse
+import os
+import tempfile
+
+from scripts.aws.aws_helper import get_session, is_s3
+from scripts.files.fs import find_sidecars, write_all
+from scripts.files.fs_s3 import bucket_name_from_path, list_uri
+from scripts.gdal.gdal_helper import run_gdal
+
+
+def main() -> None:
+    """Translates ascii files in a given path to tiffs and writes to target,
+
+    Arguments:
+    --source - local or s3 path to source data
+    --target - local or s3 path to write converted tiffs
+
+    examples:
+        python translate_ascii.py --source s3://linz-elevation-staging/test/sample-tests/ --target /tmp/
+        python translate_ascii.py --source ./tests/data/ --target /tmp/
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source", dest="source", required=True, help="S3 path to the input ascii files")
+    parser.add_argument("--target", dest="target", required=True, help="Output location path")
+    arguments = parser.parse_args()
+
+    source = arguments.source
+
+    with tempfile.TemporaryDirectory() as tmp_path:
+        if is_s3(source):
+            # list ascii files from s3
+            s3 = get_session(source).client("s3")
+            asc_files = list_uri(source, s3, extension=".asc")
+            # reformat to full s3 path
+            asc_files = [f"s3://{bucket_name_from_path(source)}/{file}" for file in asc_files]
+            # download ascii files
+            asc_files = write_all(inputs=asc_files, target=tmp_path)
+
+        else:
+            # list ascii files from local
+            asc_files = [os.path.join(source, file) for file in os.listdir(source) if file.endswith(".asc")]
+
+        tiffs = []
+        for asc in asc_files:
+            # translate from ascii to geotiff using GDAL
+            tiff = os.path.join(tmp_path, f"{asc.split('/')[-1].split('.')[0]}.tiff")
+            command = ["gdal_translate", "-of", "GTiff", asc, tiff]
+            run_gdal(command)
+            tiffs.append(tiff)
+        write_all(inputs=tiffs, target=arguments.target)
+
+        # copy any sidecar files to target
+        sidecars = find_sidecars(asc_files, [".prj", ".tfw"])
+        write_all(inputs=sidecars, target=arguments.target)
+
+
+if __name__ == "__main__":
+    main()