Merge pull request #18 from oceanmodeling/main

Updating the main repo from the dev repo

.github/workflows/tests.yml

@@ -0,0 +1,36 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '**.py'
+      - '.github/workflows/tests.yml'
+      - 'pyproject.toml'
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ '3.9', '3.10', '3.11' ]
+    steps:
+      - name: clone repository
+        uses: actions/checkout@v4
+      - name: conda virtual environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          init-shell: bash
+          environment-file: environment.yml
+      - name: install the package
+        run: pip install ".[dev]"
+        shell: micromamba-shell {0}
+      - name: run tests
+        run: pytest
+        shell: micromamba-shell {0}

README.md

@@ -8,7 +8,16 @@ TO BE COMPLETED...
 
 
 ## References
-- Daneshvar, F., et al. Tech Report (TODO)
+- Daneshvar, F., Mani, S., Pringle, W. J., Moghimi, S., Myers, E., (2024).
+_Probabilistic Prediction of Tropical Cyclone (TC) Driven Flood Depth_ 
+_and Extent with a User-Friendly Python Module._ 
+NOAA Technical Memorandum NOS CS 59. https://repository.library.noaa.gov/view/noaa/66123
+
+- Daneshvar, F., Mani, S., Pringle, W. J., Moghimi, S., Myers, E., ( 2023). 
+_Evaluating the effect of inland hydrology and hurricane model on
+the probabilistic prediction of tropical cyclone (TC) driven coastal flooding._
+American Geophysical Union (AGU) Fall Meeting, December 2023, San Francisco, CA.
+
 - Pringle, W. J., Mani, S., Sargsyan, K., Moghimi, S., Zhang, Y. J.,
 Khazaei, B., Myers, E. (January 2023).
 _Surrogate-Assisted Bayesian Uncertainty Quantification for

environment.yml

@@ -14,7 +14,7 @@ dependencies:
   - fiona
   - gdal
   - geoalchemy2
-  - geopandas>=0.13
+  - geopandas
   - geos
   - hdf5
   - importlib_metadata<8  # Fix issue with esmpy Author import
@@ -30,7 +30,7 @@ dependencies:
   - pyarrow
   - pygeos
   - pyproj
-  - python<3.11
+  - python<3.12
   - pytz
   - shapely>=2
   - rasterio
@@ -42,4 +42,4 @@ dependencies:
   - tqdm
   - udunits2
   - utm
-  - xarray==2023.7.0
+  - xarray

pyproject.toml

@@ -21,7 +21,7 @@ description = "A set of scripts to generate probabilistic storm surge results!"
 
 license = {file = "LICENSE"}
 
-requires-python = ">= 3.8, < 3.11"
+requires-python = ">= 3.9, < 3.12"
 
 dependencies = [
     "cartopy",
@@ -30,22 +30,22 @@ dependencies = [
     "cfgrib",
     "cfunits",
     "chaospy>=4.2.7",
-    "coupledmodeldriver>=1.6.6",
+    "coupledmodeldriver>=1.7.1.post1",
     "colored-traceback",
     "cmocean",
     "dask",
     "dask-jobqueue",
-    "ensembleperturbation>=1.1.2",
+    "ensembleperturbation>=1.3.4",  # perturb feature
     "fiona",
     "geoalchemy2",
-    "geopandas==0.14",  # to address AttributeError. Should be fixed later in EnsemblePerturbation
-#    "geopandas",
+    "geopandas",
     "matplotlib",
     "mpi4py",
     "netCDF4",
     "numpy",
     "numba",
     "ocsmesh==1.5.3",
+    "packaging",
     "pandas",
     "pyarrow",
     "pygeos",
@@ -54,7 +54,7 @@ dependencies = [
     "pytz",
     "pyyaml",
     "shapely>=2",
-    "stormevents==2.2.4",
+    "stormevents>=2.3.4",  # rmax option
     "rasterio",
     "requests",
     "rtree",
@@ -63,7 +63,12 @@ dependencies = [
     "typing-extensions",
     "tqdm",
     "utm",
-    "xarray==2023.7.0",
+    "xarray",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest"
 ]
 
 [tool.setuptools_scm]
@@ -94,4 +99,4 @@ download_data = "stormworkflow.prep.download_data:cli"
 setup_ensemble = "stormworkflow.prep.setup_ensemble:cli"
 combine_ensemble = "stormworkflow.post.combine_ensemble:cli"
 analyze_ensemble = "stormworkflow.post.analyze_ensemble:cli"
-storm_roc_curve = "stormworkflow.post.ROC_single_run:cli"
+storm_roc_curve = "stormworkflow.post.storm_roc_curve:cli"

stormworkflow/main.py

@@ -2,20 +2,113 @@
 import logging
 import os
 import shlex
+import warnings
 from importlib.resources import files
 from argparse import ArgumentParser
 from pathlib import Path
 
-import stormworkflow
 import yaml
+from packaging.version import Version
 try:
     from yaml import CLoader as Loader, CDumper as Dumper
 except ImportError:
     from yaml import Loader, Dumper
 
+import stormworkflow
 
 _logger = logging.getLogger(__file__)
 
+CUR_INPUT_VER = Version('0.0.5')
+VER_UPDATE_FUNCS = []
+
+
+def _input_version(prev, curr):
+    def decorator(handler):
+        def wrapper(inout_conf):
+            ver = Version(inout_conf['input_version'])
+
+            # Only update config if specified version matches the
+            # assumed one
+            if ver != Version(prev):
+                return ver
+
+            # TODO: Need return values?
+            handler(inout_conf)
+
+            return Version(curr)
+        global VER_UPDATE_FUNCS
+        VER_UPDATE_FUNCS.append(wrapper)
+        return wrapper
+    return decorator
+
+
+@_input_version('0.0.1', '0.0.2')
+def _handle_input_v0_0_1_to_v0_0_2(inout_conf):
+
+    _logger.info(
+        "Adding perturbation variables for persistent RMW perturbation"
+    )
+    inout_conf['perturb_vars'] = [
+      'cross_track',
+      'along_track',
+      'radius_of_maximum_winds_persistent',
+      'max_sustained_wind_speed',
+    ]
+
+
+@_input_version('0.0.2', '0.0.3')
+def _handle_input_v0_0_2_to_v0_0_3(inout_conf):
+
+    _logger.info(
+        "Adding RMW fill method default to persistent"
+    )
+    inout_conf['rmw_fill_method'] = 'persistent'
+
+
+@_input_version('0.0.3', '0.0.4')
+def _handle_input_v0_0_3_to_v0_0_4(inout_conf):
+
+    _logger.info(
+        "Path to observations"
+    )
+    inout_conf['NHC_OBS'] = ''
+
+
+@_input_version('0.0.4', '0.0.5')
+def _handle_input_v0_0_4_to_v0_0_5(inout_conf):
+
+    _logger.info("Adding perturbation features")
+    inout_conf['perturb_features'] = [
+      'isotach_adjustment',
+    ]
+
+
+def handle_input_version(inout_conf):
+
+    if 'input_version' not in inout_conf:
+        ver = CUR_INPUT_VER
+        warnings.warn(
+            f"`input_version` is NOT specified in `input.yaml`; assuming {ver}"
+        )
+        inout_conf['input_version'] = str(ver)
+        return
+
+    ver = Version(inout_conf['input_version'])
+
+    if ver > CUR_INPUT_VER:
+        raise ValueError(
+            f"Input version not supported! Max version supported is {CUR_INPUT_VER}"
+        )
+
+    for fn in VER_UPDATE_FUNCS:
+        ver = fn(inout_conf)
+        inout_conf['input_version'] = str(ver)
+
+    if ver != CUR_INPUT_VER:
+        raise ValueError(
+            f"Could NOT update input to the latest version! Updated to {ver}"
+        )
+
 def main():
 
     parser = ArgumentParser()
@@ -28,12 +121,17 @@ def main():
 
     infile = args.configuration
     if infile is None:
-        _logger.warn('No input configuration provided, using reference file!')
+        warnings.warn(
+            'No input configuration provided, using reference file!'
+        )
         infile = refs.joinpath('input.yaml')
 
     with open(infile, 'r') as yfile:
         conf = yaml.load(yfile, Loader=Loader)
 
+    handle_input_version(conf)
+    # TODO: Write out the updated config as a yaml file
+
     wf = scripts.joinpath('workflow.sh')
 
     run_env = os.environ.copy()

stormworkflow/post/analyze_ensemble.py

@@ -408,19 +408,19 @@ def cli():
     parser.add_argument('-t', '--tracks-dir', type=Path)
     parser.add_argument('-s', '--sequential', action='store_true')
 
-    main(parser.parse_args())
-
-
-if __name__ == '__main__':
     cluster = SLURMCluster(cores=16,
                            processes=1,
                            memory="500GB",
-                           account="compute",
+                           account="nos-surge", #PW:"compute" ; Hercules:"nos-surge" or "nosofs"
                            walltime="08:00:00",
-                           header_skip=['--mem'],
-                           interface="eth0") 
-    cluster.scale(6) 
-    client = Client(cluster) 
+                           header_skip=['--mem']) #,
+#                           interface="eth0")     # only for PW
+    cluster.scale(6)
+    client = Client(cluster)
     print(client)
 
+    main(parser.parse_args())
+
+
+if __name__ == '__main__':
     cli()