Merge pull request #185 from jarq6c/nwis-client-cli

Add `nwis_client` CLI

python/nwis_client/README.md

@@ -54,3 +54,95 @@ print(observations_data.head())
 3 2019-08-01 04:45:00    streamflow       01646500            ft3/s  4170.0        [A]       0
 4 2019-08-01 05:00:00    streamflow       01646500            ft3/s  4170.0        [A]       0
 ```
+
+### Command Line Interface (CLI)
+The `hydrotools.nwis_client` package includes a command-line utility.
+
+This example demonstrates calling the help page:
+```bash
+$ nwis-client --help
+```
+```console
+Usage: nwis-client [OPTIONS] [SITES]...
+
+  Retrieve data from the USGS IV Web Service API and write in CSV format.
+
+  Example:
+
+  nwis-client 01013500 02146470
+
+Options:
+  -o, --output FILENAME       Output file path
+  -s, --startDT TIMESTAMP     Start datetime
+  -e, --endDT TIMESTAMP       End datetime
+  -p, --parameterCd TEXT      Parameter code
+  --comments / --no-comments  Enable/disable comments in output, enabled by
+                              default
+  --header / --no-header      Enable/disable header in output, enabled by
+                              default
+  --help                      Show this message and exit.
+```
+
+This example retrieves the last discharge value from two sites:
+```bash
+$ nwis-client 01013500 02146470
+```
+```console
+# USGS IV Service Data
+# 
+# value_date: Datetime of measurement (UTC) (character string)
+# variable: USGS variable name (character string)
+# usgs_site_code: USGS Gage Site Code (character string)
+# measurement_unit: Units of measurement (character string)
+# value: Measurement value (float)
+# qualifiers: Qualifier string (character string)
+# series: Series number in case multiple time series are returned (integer)
+# 
+# Generated at 2022-03-04 21:56:30.296051+00:00
+# nwis_client version: 3.2.0
+# Source code: https://github.com/NOAA-OWP/hydrotools
+# 
+value_time,variable_name,usgs_site_code,measurement_unit,value,qualifiers,series
+2022-03-04 21:45:00,streamflow,01013500,ft3/s,-999999.00,"['P', 'Ice']",0
+2022-03-04 21:50:00,streamflow,02146470,ft3/s,1.04,['P'],0
+```
+
+This example retrieves stage data from two sites for a specific time period:
+```bash
+$ nwis-client -p 00065 -s 2021-06-01T00:00 -e 2021-06-01T01:00 01013500 02146470
+```
+```console
+# USGS IV Service Data
+# 
+# value_date: Datetime of measurement (UTC) (character string)
+# variable: USGS variable name (character string)
+# usgs_site_code: USGS Gage Site Code (character string)
+# measurement_unit: Units of measurement (character string)
+# value: Measurement value (float)
+# qualifiers: Qualifier string (character string)
+# series: Series number in case multiple time series are returned (integer)
+# 
+# Generated at 2022-03-04 21:59:02.508468+00:00
+# nwis_client version: 3.2.0
+# Source code: https://github.com/NOAA-OWP/hydrotools
+# 
+value_time,variable_name,usgs_site_code,measurement_unit,value,qualifiers,series
+2021-05-31 23:00:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:15:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:30:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:45:00,gage height,01013500,ft,4.28,['A'],0
+2021-06-01 00:00:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:00:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:05:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:10:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:15:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:20:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:25:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:30:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:35:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:40:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:45:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:50:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:55:00,gage height,02146470,ft,3.14,['A'],0
+2021-06-01 00:00:00,gage height,02146470,ft,3.14,['A'],0
+```

python/nwis_client/setup.cfg

@@ -34,6 +34,7 @@ install_requires =
     numpy
     hydrotools._restclient>=3.0.4
     aiohttp
+    click
 python_requires = >=3.7
 include_package_data = True
 
@@ -45,3 +46,7 @@ develop =
     pytest
     pytest-aiohttp
 
+[options.entry_points]
+console_scripts =
+    nwis-client = hydrotools.nwis_client.cli:run
+

python/nwis_client/src/hydrotools/nwis_client/_version.py

@@ -1 +1 @@
-__version__ = "3.1.0"
+__version__ = "3.2.0"

python/nwis_client/src/hydrotools/nwis_client/cli.py

@@ -0,0 +1,102 @@
+import click
+from hydrotools.nwis_client import IVDataService
+from hydrotools.nwis_client import _version as CLIENT_VERSION
+from typing import Tuple
+import pandas as pd
+from dataclasses import dataclass
+
+class TimestampParamType(click.ParamType):
+    name = "timestamp"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, pd.Timestamp):
+            return value
+
+        try:
+            return pd.Timestamp(value)
+        except ValueError:
+            self.fail(f"{value!r} is not a valid timestamp", param, ctx)
+
+@dataclass
+class CSVDefaults:
+    comments: str = """# USGS IV Service Data
+# 
+# value_date: Datetime of measurement (UTC) (character string)
+# variable: USGS variable name (character string)
+# usgs_site_code: USGS Gage Site Code (character string)
+# measurement_unit: Units of measurement (character string)
+# value: Measurement value (float)
+# qualifiers: Qualifier string (character string)
+# series: Series number in case multiple time series are returned (integer)
+# 
+"""
+
+def write_to_csv(
+    data: pd.DataFrame,
+    ofile: click.File,
+    comments: bool = True,
+    header: bool = True
+    ) -> None:
+    # Get default options
+    defaults = CSVDefaults()
+
+    # Comments
+    if comments:
+        output = defaults.comments
+
+        # Add version, link, and write time
+        now = pd.Timestamp.utcnow()
+        output += f"# Generated at {now}\n"
+        output += f"# nwis_client version: {CLIENT_VERSION.__version__}\n"
+        output += "# Source code: https://github.com/NOAA-OWP/hydrotools\n# \n"
+
+        # Write comments to file
+        ofile.write(output)
+
+    # Write data to file
+    data.to_csv(ofile, mode="a", index=False, float_format="{:.2f}".format, header=header, chunksize=20000)
+
+@click.command()
+@click.argument("sites", nargs=-1, required=False)
+@click.option("-o", "--output", nargs=1, type=click.File("w"), help="Output file path", default="-")
+@click.option("-s", "--startDT", "startDT", nargs=1, type=TimestampParamType(), help="Start datetime")
+@click.option("-e", "--endDT", "endDT", nargs=1, type=TimestampParamType(), help="End datetime")
+@click.option("-p", "--parameterCd", "parameterCd", nargs=1, type=str, default="00060", help="Parameter code")
+@click.option('--comments/--no-comments', default=True, help="Enable/disable comments in output, enabled by default")
+@click.option('--header/--no-header', default=True, help="Enable/disable header in output, enabled by default")
+def run(
+    sites: Tuple[str], 
+    output: click.File, 
+    startDT: pd.Timestamp = None,
+    endDT: pd.Timestamp = None,
+    parameterCd: str = "00060",
+    comments: bool = True,
+    header: bool = True
+    ) -> None:
+    """Retrieve data from the USGS IV Web Service API and write in CSV format.
+
+    Example:
+    
+    nwis-client 01013500 02146470
+    """
+    # Get sites
+    if not sites:
+        print("Reading sites from stdin: ")
+        sites = click.get_text_stream("stdin").read().split()
+
+    # Setup client
+    client = IVDataService(value_time_label="value_time")
+
+    # Retrieve data
+    df = client.get(
+        sites=sites,
+        startDT=startDT,
+        endDT=endDT,
+        parameterCd=parameterCd
+    )
+
+    # Write to CSV
+    write_to_csv(data=df, ofile=output, comments=comments, header=header)
+
+if __name__ == "__main__":
+    run()

python/nwis_client/src/hydrotools/nwis_client/iv.py

@@ -35,8 +35,7 @@
 from ._utilities import verify_case_insensitive_kwargs
 
 def _verify_case_insensitive_kwargs_handler(m: str) -> None:
-    warnings.warn("`hydrotools.nwis_client` > 3.1 will raise RuntimeError exception instead of RuntimeWarning.", DeprecationWarning)
-    warnings.warn(m, RuntimeWarning)
+    raise RuntimeError(m)
 
 class IVDataService:
     """
@@ -51,7 +50,7 @@ class IVDataService:
         Toggle sqlite3 request caching
     cache_expire_after : int
         Cached item life length in seconds
-    value_time_label: str, default 'value_date'
+    value_time_label: str, default 'value_time'
         Label to use for datetime column returned by IVDataService.get
     cache_filename: str or Path default 'nwisiv_cache'
         Sqlite cache filename or filepath. Suffix '.sqlite' will be added to file if not included.
@@ -106,7 +105,7 @@ class IVDataService:
     def __init__(self, *, 
         enable_cache: bool = True, 
         cache_expire_after: int = 43200,
-        value_time_label: str = None,
+        value_time_label: str = "value_time",
         cache_filename: Union[str, Path] = "nwisiv_cache"
         ):
         self._cache_enabled = enable_cache

python/nwis_client/tests/test_client_cli.py

@@ -0,0 +1,45 @@
+import pytest
+from hydrotools.nwis_client import cli
+from pathlib import Path
+import subprocess
+from tempfile import TemporaryDirectory
+
+def test_cli():
+    """Normaly would use click.testing.CLiRunner. However, this does not appear to be async friendly."""
+    with TemporaryDirectory() as tdir:
+        # Test default parameters
+        result1 = subprocess.run([
+            "nwis-client", "01013500", "02146470", "-o", f"{tdir}/test_output_1.csv"
+        ])
+        assert result1.returncode == 0
+        assert Path(f"{tdir}/test_output_1.csv").exists()
+
+        # Test other parameters
+        result2 = subprocess.run([
+            "nwis-client",
+            '-s', '2022-01-01',
+            '-e', '2022-01-02',
+            '-p', '00065',
+            '01013500', '02146470', "-o", f"{tdir}/test_output_2.csv"])
+        assert result2.returncode == 0
+        assert Path(f"{tdir}/test_output_2.csv").exists()
+
+def test_comments_header():
+    """Normaly would use click.testing.CLiRunner. However, this does not appear to be async friendly."""
+    with TemporaryDirectory() as tdir:
+        # Output file
+        ofile = Path(tdir) / "test_output.csv"
+
+        # Disable comments and header
+        result2 = subprocess.run([
+            "nwis-client",
+            '--no-comments', '--no-header',
+            '01013500', '02146470', "-o", str(ofile)])
+        assert result2.returncode == 0
+        assert ofile.exists()
+
+        # File should only have two lines
+        with ofile.open('r') as fi:
+            count = len([l for l in fi])
+            assert count == 2
+

python/nwis_client/tests/test_nwis.py

@@ -272,7 +272,7 @@ def test_get_slow(setup_iv):
     df = setup_iv.get(site, startDT=start, endDT=end)
     # IV api seems to send different start based on daylights saving time.
     # Test is less prescriptive, but still should suffice
-    assert df["value_date"][0].isoformat().startswith(start)
+    assert df["value_time"][0].isoformat().startswith(start)
 
 
 datetime_keyword_test_data_should_fail = [
@@ -479,10 +479,10 @@ def test_nwis_client_get_throws_warning_for_kwargs(mocked_iv):
     version = (version.major, version.minor)
 
     # versions > 3.1 should throw an exception instead of a warning
-    assert version <= (3, 1)
+    assert version > (3, 1)
 
-    with pytest.warns(RuntimeWarning, match="function parameter, 'startDT', provided as 'startDt'"):
-        # startdt should be startDT
+    with pytest.raises(RuntimeError):
+        # startDt should be startDT
         mocked_iv.get(sites=["01189000"], startDt="2022-01-01")
 
 @pytest.mark.slow