remove api keys/new elevation downloader

dev-requirements.txt

@@ -48,6 +48,7 @@ click==8.1.7
     #   cligj
     #   fiona
     #   pip-tools
+    #   planetary-computer
     #   rasterio
 click-plugins==1.1.1
     # via
@@ -109,7 +110,9 @@ iniconfig==2.0.0
 joblib==1.4.2
     # via swmmanywhere (pyproject.toml)
 jsonschema==4.22.0
-    # via swmmanywhere (pyproject.toml)
+    # via
+    #   pystac
+    #   swmmanywhere (pyproject.toml)
 jsonschema-specifications==2023.12.1
     # via jsonschema
 julian==0.14
@@ -120,8 +123,6 @@ llvmlite==0.42.0
     # via numba
 loguru==0.7.2
     # via swmmanywhere (pyproject.toml)
-looseversion==1.3.0
-    # via pysheds
 multiurl==0.3.1
     # via cads-api-client
 mypy==1.10.0
@@ -177,6 +178,7 @@ packaging==24.1
     #   fastparquet
     #   geopandas
     #   lazy-loader
+    #   planetary-computer
     #   pyswmm
     #   pytest
     #   rioxarray
@@ -196,6 +198,8 @@ pillow==10.3.0
     #   scikit-image
 pip-tools==7.4.1
     # via swmmanywhere (pyproject.toml)
+planetary-computer==1.0.0
+    # via swmmanywhere (pyproject.toml)
 platformdirs==4.2.2
     # via virtualenv
 pluggy==1.5.0
@@ -205,7 +209,9 @@ pre-commit==3.7.1
 pyarrow==16.1.0
     # via swmmanywhere (pyproject.toml)
 pydantic==2.7.3
-    # via swmmanywhere (pyproject.toml)
+    # via
+    #   planetary-computer
+    #   swmmanywhere (pyproject.toml)
 pydantic-core==2.18.4
     # via pydantic
 pyflwdir==0.5.8
@@ -223,6 +229,14 @@ pyproject-hooks==1.1.0
     #   pip-tools
 pysheds==0.3.5
     # via swmmanywhere (pyproject.toml)
+pystac[validation]==1.10.1
+    # via
+    #   planetary-computer
+    #   pystac-client
+pystac-client==0.8.2
+    # via
+    #   planetary-computer
+    #   swmmanywhere (pyproject.toml)
 pyswmm==2.0.1
     # via swmmanywhere (pyproject.toml)
 pytest==8.2.2
@@ -241,10 +255,15 @@ python-dateutil==2.9.0.post0
     # via
     #   multiurl
     #   pandas
+    #   pystac
+    #   pystac-client
+python-dotenv==1.0.1
+    # via planetary-computer
 pytz==2024.1
     # via
     #   multiurl
     #   pandas
+    #   planetary-computer
 pyyaml==6.0.1
     # via
     #   pre-commit
@@ -264,6 +283,8 @@ requests==2.32.3
     #   cdsapi
     #   multiurl
     #   osmnx
+    #   planetary-computer
+    #   pystac-client
 rioxarray==0.15.5
     # via swmmanywhere (pyproject.toml)
 rpds-py==0.18.1

pyproject.toml

@@ -39,10 +39,12 @@ dependencies = [
   "numpy",
   "osmnx",
   "pandas",
+  "planetary_computer",
   "pyarrow",
   "pydantic",
   "pyflwdir",
   "pysheds",
+  "pystac_client",
   "pyswmm",
   "PyYAML",
   "rasterio",

requirements.txt

@@ -39,6 +39,7 @@ click==8.1.7
     #   click-plugins
     #   cligj
     #   fiona
+    #   planetary-computer
     #   rasterio
 click-plugins==1.1.1
     # via
@@ -86,7 +87,9 @@ imageio==2.33.1
 joblib==1.3.2
     # via swmmanywhere (pyproject.toml)
 jsonschema==4.21.1
-    # via swmmanywhere (pyproject.toml)
+    # via
+    #   pystac
+    #   swmmanywhere (pyproject.toml)
 jsonschema-specifications==2023.12.1
     # via jsonschema
 julian==0.14
@@ -140,6 +143,7 @@ packaging==23.2
     #   fastparquet
     #   geopandas
     #   lazy-loader
+    #   planetary-computer
     #   pyswmm
     #   rioxarray
     #   scikit-image
@@ -156,10 +160,14 @@ pillow==10.2.0
     # via
     #   imageio
     #   scikit-image
+planetary-computer==1.0.0
+    # via swmmanywhere (pyproject.toml)
 pyarrow==14.0.2
     # via swmmanywhere (pyproject.toml)
 pydantic==2.5.3
-    # via swmmanywhere (pyproject.toml)
+    # via
+    #   planetary-computer
+    #   swmmanywhere (pyproject.toml)
 pydantic-core==2.14.6
     # via pydantic
 pyflwdir==0.5.8
@@ -173,12 +181,27 @@ pyproj==3.6.1
     #   rioxarray
 pysheds==0.3.5
     # via swmmanywhere (pyproject.toml)
+pystac[validation]==1.10.1
+    # via
+    #   planetary-computer
+    #   pystac-client
+pystac-client==0.8.2
+    # via
+    #   planetary-computer
+    #   swmmanywhere (pyproject.toml)
 pyswmm==1.5.1
     # via swmmanywhere (pyproject.toml)
 python-dateutil==2.9.0.post0
-    # via pandas
+    # via
+    #   pandas
+    #   pystac
+    #   pystac-client
+python-dotenv==1.0.1
+    # via planetary-computer
 pytz==2024.1
-    # via pandas
+    # via
+    #   pandas
+    #   planetary-computer
 pyyaml==6.0.1
     # via swmmanywhere (pyproject.toml)
 rasterio==1.3.9
@@ -194,6 +217,8 @@ requests==2.31.0
     # via
     #   cdsapi
     #   osmnx
+    #   planetary-computer
+    #   pystac-client
 rioxarray==0.15.1
     # via swmmanywhere (pyproject.toml)
 rpds-py==0.18.0

swmmanywhere/defs/schema.yml

@@ -3,7 +3,6 @@ properties:
   base_dir: {type: string}
   project: {type: string}
   bbox: {type: array, items: {type: number}, minItems: 4, maxItems: 4}
-  api_keys: {type: string}
   model_number: {type: integer}
   run_settings:
     type: object
@@ -31,4 +30,4 @@ properties:
   metric_list: {type: array, items: {type: string}}
   address_overrides: {type: ['object', 'null']}
   parameter_overrides: {type: ['object', 'null']}
-required: [base_dir, project, bbox, api_keys, graphfcn_list]
+required: [base_dir, project, bbox, graphfcn_list]

swmmanywhere/prepare_data.py

@@ -4,15 +4,18 @@
 """
 from __future__ import annotations
 
-import shutil
 from pathlib import Path
 from typing import cast
 
 import cdsapi
 import networkx as nx
 import osmnx as ox
 import pandas as pd
+import planetary_computer
+import pystac_client
 import requests
+import rioxarray
+import rioxarray.merge as rxr_merge
 import xarray as xr
 from geopy.geocoders import Nominatim
 
@@ -138,56 +141,46 @@ def download_river(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph:
     return cast("nx.MultiDiGraph", graph)
 
 def download_elevation(fid: Path, 
-                       bbox: tuple[float, float, float, float], 
-                       api_key: str ='<your_api_key>') -> int:
-    """Download NASADEM elevation data from OpenTopography API.
+                       bbox: tuple[float, float, float, float]) -> int:
+    """Download NASADEM elevation data from Microsoft Planetary computer.
 
-    Downloads elevation data in GeoTIFF format from OpenTopography API based on
-      the specified bounding box.
+    Downloads elevation data in GeoTIFF format from Microsoft Planetary computer
+      based on the specified bounding box.
 
     Args:
         fid (Path): File path to save the downloaded elevation data.
         bbox (tuple): Bounding box coordinates in the format 
             (minx, miny, maxx, maxy).
-        api_key (str, optional): Your OpenTopography API key. 
-            Defaults to '<your_api_key>'.
 
     Returns:
-        status_code (int): Response status code
-
-    Raises:
-        requests.exceptions.RequestException: If there is an error in the API 
-            request.
+        status_code (int): Response status code.
 
     Example:
         ```
         bbox = (-120, 35, -118, 37)  # Example bounding box coordinates
         download_elevation('elevation_data.tif', 
-                            bbox, 
-                            api_key='your_actual_api_key')
+                            bbox)
         ```
 
-    Note:
-        To obtain an API key, you need to sign up on the OpenTopography 
-        website.
-
+    Author:
+        cheginit
     """
-    minx, miny, maxx, maxy = bbox
-    url = f'https://portal.opentopography.org/API/globaldem?demtype=NASADEM&south={miny}&north={maxy}&west={minx}&east={maxx}&outputFormat=GTiff&API_Key={api_key}'
-
-    try:
-        r = requests.get(url, stream=True)
-        r.raise_for_status()
-
-        with fid.open('wb') as rast_file:
-            shutil.copyfileobj(r.raw, rast_file)
-            
-        logger.info('Elevation data downloaded successfully.')
-
-    except requests.exceptions.RequestException as e:
-        logger.error(f'Error downloading elevation data: {e}')
-
-    return r.status_code
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+        modifier=planetary_computer.sign_inplace,
+    )
+    search = catalog.search(
+        collections=["nasadem"],
+        bbox=bbox,
+    )
+    items = list(search.items())
+    signed_asset = [planetary_computer.sign(item.assets["elevation"]).href 
+                    for item in items]
+    dem = rxr_merge.merge_arrays([rioxarray.open_rasterio(href).squeeze(drop=True) 
+                                  for href in signed_asset])
+    dem = dem.rio.clip_box(*bbox)
+    dem.rio.to_raster(fid)
+    return 200
 
 def download_precipitation(bbox: tuple[float, float, float, float], 
                            start_date: str = '2015-01-01',

swmmanywhere/preprocessing.py

@@ -182,7 +182,6 @@ def prepare_precipitation(bbox: tuple[float, float, float, float],
 
 def prepare_elevation(bbox: tuple[float, float, float, float],
                     addresses: parameters.FilePaths,
-                    api_keys: dict[str, str],
                     target_crs: str):
     """Download and reproject elevation data."""
     if addresses.elevation.exists():
@@ -192,7 +191,6 @@ def prepare_elevation(bbox: tuple[float, float, float, float],
         fid = Path(temp_dir) / 'elevation.tif'
         prepare_data.download_elevation(fid,
                                         bbox, 
-                                        api_keys['nasadem_key']
                                         )
         go.reproject_raster(target_crs,
                             fid,
@@ -280,7 +278,6 @@ def prepare_river(bbox: tuple[float, float, float, float],
 
 def run_downloads(bbox: tuple[float, float, float, float],
                   addresses: parameters.FilePaths,
-                  api_keys: dict[str, str],
                   network_types = ['drive']):
     """Run the data downloads.
 
@@ -292,16 +289,16 @@ def run_downloads(bbox: tuple[float, float, float, float],
         bbox (tuple[float, float, float, float]): Bounding box coordinates in 
             the format (minx, miny, maxx, maxy) in EPSG:4326.
         addresses (FilePaths): Class containing the addresses of the directories.
-        api_keys (dict): Dictionary containing the API keys.
         network_types (list): List of network types to download.
     """
     target_crs = go.get_utm_epsg(bbox[0], bbox[1])
 
     # Download precipitation data
-    prepare_precipitation(bbox, addresses, api_keys, target_crs)
+    # Currently commented because it doesn't work
+    # prepare_precipitation(bbox, addresses, api_keys, target_crs)
 
     # Download elevation data
-    prepare_elevation(bbox, addresses, api_keys, target_crs)
+    prepare_elevation(bbox, addresses, target_crs)
 
     # Download building data
     prepare_building(bbox, addresses, target_crs)

swmmanywhere/swmmanywhere.py

@@ -65,10 +65,8 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]:
 
     # Run downloads
     logger.info("Running downloads.")
-    api_keys = yaml_load(config['api_keys'].read_text())
     preprocessing.run_downloads(config['bbox'],
                 addresses,
-                api_keys,
                 network_types = params['topology_derivation'].allowable_networks
                 )
 
@@ -152,7 +150,7 @@ def check_top_level_paths(config: dict):
     Raises:
         FileNotFoundError: If a top level path does not exist.
     """
-    for key in ['base_dir', 'api_keys']:
+    for key in ['base_dir']:
         if not Path(config[key]).exists():
             raise FileNotFoundError(f"{key} not found at {config[key]}")
         config[key] = Path(config[key])

tests/test_data/demo_config.yml

@@ -1,7 +1,6 @@
 base_dir: /path/to/base/directory
 project: demo
 bbox: [0.04020, 51.55759, 0.09826, 51.62050]
-api_keys: /path/to/api/keys.yml
 run_settings: 
   reporting_iters: 100
   duration: 86400