From 4f6fd0e56c109561ffd5ee88c9afb578a834b492 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 15:09:36 -0600
Subject: [PATCH 01/18] add click requirement and entry point

---
 python/nwis_client/setup.cfg | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/python/nwis_client/setup.cfg b/python/nwis_client/setup.cfg
index ef5d89f9..346417a8 100644
--- a/python/nwis_client/setup.cfg
+++ b/python/nwis_client/setup.cfg
@@ -34,6 +34,7 @@ install_requires =
     numpy
     hydrotools._restclient>=3.0.4
     aiohttp
+    click
 python_requires = >=3.7
 include_package_data = True
 
@@ -45,3 +46,7 @@ develop =
     pytest
     pytest-aiohttp
     
+[options.entry_points]
+console_scripts =
+    nwis-client = hydrotools.nwis_client.cli:run
+    
\ No newline at end of file

From cc3636860608f60a2ad361580bb3ba1a7018f9e8 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 15:09:48 -0600
Subject: [PATCH 02/18] basic cli

---
 .../src/hydrotools/nwis_client/cli.py         | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 python/nwis_client/src/hydrotools/nwis_client/cli.py

diff --git a/python/nwis_client/src/hydrotools/nwis_client/cli.py b/python/nwis_client/src/hydrotools/nwis_client/cli.py
new file mode 100644
index 00000000..1ca03fae
--- /dev/null
+++ b/python/nwis_client/src/hydrotools/nwis_client/cli.py
@@ -0,0 +1,82 @@
+import click
+from hydrotools.nwis_client import IVDataService
+from hydrotools.nwis_client import _version as CLIENT_VERSION
+from typing import Tuple
+from pathlib import Path
+import pandas as pd
+
+class TimestampParamType(click.ParamType):
+    name = "timestamp"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, pd.Timestamp):
+            return value
+
+        try:
+            return pd.Timestamp(value)
+        except ValueError:
+            self.fail(f"{value!r} is not a valid timestamp", param, ctx)
+
+def write_to_csv(
+    data: pd.DataFrame,
+    ofile: Path
+    ) -> None:
+    # Comments
+    output = """# USGS IV Service Data
+# 
+# value_date: Datetime of measurement (UTC) (character string)
+# variable: USGS variable name (character string)
+# usgs_site_code: USGS Gage Site Code (character string)
+# measurement_unit: Units of measurement (character string)
+# value: Measurement value (float)
+# qualifiers: Qualifier string (character string)
+# series: Series number in case multiple time series are returned (integer)
+# 
+"""
+    # Add version, link, and write time
+    now = pd.Timestamp.utcnow()
+    output += f"# Generated at {now}\n"
+    output += f"# nwis_client version: {CLIENT_VERSION.__version__}\n"
+    output += "# Source code: https://github.com/NOAA-OWP/hydrotools\n# \n"
+
+    # Write to file
+    output += data.to_csv(index=False, float_format="{:.2f}".format)
+    with ofile.open("w") as of:
+        of.write(output)
+
+@click.command()
+@click.argument("sites", nargs=-1)
+@click.argument("ofile", nargs=1, type=click.Path(path_type=Path))
+@click.option("-s", "--startDT", "startDT", nargs=1, type=TimestampParamType(), help="Start datetime")
+@click.option("-e", "--endDT", "endDT", nargs=1, type=TimestampParamType(), help="End datetime")
+@click.option("-p", "--parameterCd", "parameterCd", nargs=1, type=str, help="Parameter code")
+def run(
+    sites: Tuple[str], 
+    ofile: Path, 
+    startDT: pd.Timestamp = None,
+    endDT: pd.Timestamp = None,
+    parameterCd: str = None
+    ) -> None:
+    """Retrieve data from the USGS IV Web Service API and write to CSV.
+    Writes data for all SITES to OFILE.
+
+    Example:
+    
+    nwis-client 01013500 02146470 my_output.csv
+    """
+    # Setup client
+    client = IVDataService(value_time_label="value_time")
+
+    # Retrieve data
+    df = client.get(
+        sites=sites,
+        startDT=startDT,
+        endDT=endDT,
+        parameterCd=parameterCd
+    )
+    
+    # Write to CSV
+    write_to_csv(data=df, ofile=ofile)
+
+if __name__ == "__main__":
+    run()

From 330f8cb62d9444e998cca743e24abdc70817647b Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 15:47:22 -0600
Subject: [PATCH 03/18] update default parameter code

---
 python/nwis_client/src/hydrotools/nwis_client/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/cli.py b/python/nwis_client/src/hydrotools/nwis_client/cli.py
index 1ca03fae..fb0b4eb0 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/cli.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/cli.py
@@ -49,13 +49,13 @@ def write_to_csv(
 @click.argument("ofile", nargs=1, type=click.Path(path_type=Path))
 @click.option("-s", "--startDT", "startDT", nargs=1, type=TimestampParamType(), help="Start datetime")
 @click.option("-e", "--endDT", "endDT", nargs=1, type=TimestampParamType(), help="End datetime")
-@click.option("-p", "--parameterCd", "parameterCd", nargs=1, type=str, help="Parameter code")
+@click.option("-p", "--parameterCd", "parameterCd", nargs=1, type=str, default="00060", help="Parameter code")
 def run(
     sites: Tuple[str], 
     ofile: Path, 
     startDT: pd.Timestamp = None,
     endDT: pd.Timestamp = None,
-    parameterCd: str = None
+    parameterCd: str = "00060"
     ) -> None:
     """Retrieve data from the USGS IV Web Service API and write to CSV.
     Writes data for all SITES to OFILE.

From 1f5b27057045f029521fe21241784ab0ece4468e Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 15:47:45 -0600
Subject: [PATCH 04/18] add basic test

---
 python/nwis_client/tests/test_client_cli.py | 22 +++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 python/nwis_client/tests/test_client_cli.py

diff --git a/python/nwis_client/tests/test_client_cli.py b/python/nwis_client/tests/test_client_cli.py
new file mode 100644
index 00000000..cdabb5ad
--- /dev/null
+++ b/python/nwis_client/tests/test_client_cli.py
@@ -0,0 +1,22 @@
+import pytest
+from click.testing import CliRunner
+from hydrotools.nwis_client.cli import run
+from pathlib import Path
+
+def test_cli():
+    runner = CliRunner()
+
+    with runner.isolated_filesystem():
+        # Test default parameters
+        result1 = runner.invoke(run, ['01013500', '02146470', "test_output_1.csv"])
+        assert result1.exit_code == 0
+        assert Path("test_output_1.csv").exists()
+
+        # Test other parameters
+        result2 = runner.invoke(run, [
+            '-s', '2022-01-01',
+            '-e', '2022-01-02',
+            '-p', '00065',
+            '01013500', '02146470', "test_output_2.csv"])
+        assert result2.exit_code == 0
+        assert Path("test_output_2.csv").exists()

From 6e3a36dfd0f659d88081293ccaad1d70d1aad7dc Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 15:49:18 -0600
Subject: [PATCH 05/18] increment version

---
 python/nwis_client/src/hydrotools/nwis_client/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/_version.py b/python/nwis_client/src/hydrotools/nwis_client/_version.py
index f5f41e56..11731085 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/_version.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/_version.py
@@ -1 +1 @@
-__version__ = "3.1.0"
+__version__ = "3.2.0"

From 76c929df0d896daffda7045f99f3aa2e45c72086 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 16:00:15 -0600
Subject: [PATCH 06/18] add cli example

---
 python/nwis_client/README.md | 90 ++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/python/nwis_client/README.md b/python/nwis_client/README.md
index cf93f747..8831f5a0 100644
--- a/python/nwis_client/README.md
+++ b/python/nwis_client/README.md
@@ -54,3 +54,93 @@ print(observations_data.head())
 3 2019-08-01 04:45:00    streamflow       01646500            ft3/s  4170.0        [A]       0
 4 2019-08-01 05:00:00    streamflow       01646500            ft3/s  4170.0        [A]       0
 ```
+
+### Command Line Interface (CLI)
+The `hydrotools.nwis_client` package also includes a command-line utility that is installed to the environment targeted by the `pip` installation above. Starting a terminal and activating the correct Python environment should enable the `nwis-client` CLI. This tool will retrieve data from the USGS IV Web Service and write it to CSV format.
+
+This example demonstrates calling the help page:
+```bash
+$ nwis-client --help
+```
+```console
+Usage: nwis-client [OPTIONS] [SITES]... OFILE
+
+  Retrieve data from the USGS IV Web Service API and write to CSV. Writes data
+  for all SITES to OFILE.
+
+  Example:
+
+  nwis-client 01013500 02146470 my_output.csv
+
+Options:
+  -s, --startDT TIMESTAMP  Start datetime
+  -e, --endDT TIMESTAMP    End datetime
+  -p, --parameterCd TEXT   Parameter code
+  --help                   Show this message and exit.
+```
+
+This example retrieves the last discharge value from two sites:
+```bash
+$ nwis-client 01013500 02146470 my_data.csv
+$ cat my_data.csv
+```
+```console
+# USGS IV Service Data
+# 
+# value_date: Datetime of measurement (UTC) (character string)
+# variable: USGS variable name (character string)
+# usgs_site_code: USGS Gage Site Code (character string)
+# measurement_unit: Units of measurement (character string)
+# value: Measurement value (float)
+# qualifiers: Qualifier string (character string)
+# series: Series number in case multiple time series are returned (integer)
+# 
+# Generated at 2022-03-04 21:56:30.296051+00:00
+# nwis_client version: 3.2.0
+# Source code: https://github.com/NOAA-OWP/hydrotools
+# 
+value_time,variable_name,usgs_site_code,measurement_unit,value,qualifiers,series
+2022-03-04 21:45:00,streamflow,01013500,ft3/s,-999999.00,"['P', 'Ice']",0
+2022-03-04 21:50:00,streamflow,02146470,ft3/s,1.04,['P'],0
+```
+
+This example retrieves stage data from two sites for a specific time period:
+```bash
+$ nwis-client -p 00065 -s 2021-06-01T00:00 -e 2021-06-01T01:00 01013500 02146470 my_data.csv
+$ cat my_data.csv
+```
+```console
+# USGS IV Service Data
+# 
+# value_date: Datetime of measurement (UTC) (character string)
+# variable: USGS variable name (character string)
+# usgs_site_code: USGS Gage Site Code (character string)
+# measurement_unit: Units of measurement (character string)
+# value: Measurement value (float)
+# qualifiers: Qualifier string (character string)
+# series: Series number in case multiple time series are returned (integer)
+# 
+# Generated at 2022-03-04 21:59:02.508468+00:00
+# nwis_client version: 3.2.0
+# Source code: https://github.com/NOAA-OWP/hydrotools
+# 
+value_time,variable_name,usgs_site_code,measurement_unit,value,qualifiers,series
+2021-05-31 23:00:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:15:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:30:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:45:00,gage height,01013500,ft,4.28,['A'],0
+2021-06-01 00:00:00,gage height,01013500,ft,4.28,['A'],0
+2021-05-31 23:00:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:05:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:10:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:15:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:20:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:25:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:30:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:35:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:40:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:45:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:50:00,gage height,02146470,ft,3.14,['A'],0
+2021-05-31 23:55:00,gage height,02146470,ft,3.14,['A'],0
+2021-06-01 00:00:00,gage height,02146470,ft,3.14,['A'],0
+```

From 6e9142c40f62dd0c7637f1a295ce071e93efbf76 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 16:29:54 -0600
Subject: [PATCH 07/18] test for exception instead of warning

---
 python/nwis_client/tests/test_nwis.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/nwis_client/tests/test_nwis.py b/python/nwis_client/tests/test_nwis.py
index 60d8a1d9..e0b44c1f 100644
--- a/python/nwis_client/tests/test_nwis.py
+++ b/python/nwis_client/tests/test_nwis.py
@@ -479,10 +479,10 @@ def test_nwis_client_get_throws_warning_for_kwargs(mocked_iv):
     version = (version.major, version.minor)
 
     # versions > 3.1 should throw an exception instead of a warning
-    assert version <= (3, 1)
+    assert version > (3, 1)
 
-    with pytest.warns(RuntimeWarning, match="function parameter, 'startDT', provided as 'startDt'"):
-        # startdt should be startDT
+    with pytest.raises(RuntimeError):
+        # startDt should be startDT
         mocked_iv.get(sites=["01189000"], startDt="2022-01-01")
 
 @pytest.mark.slow

From 7793404997f7931961008b3277b379442741b20f Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 16:30:13 -0600
Subject: [PATCH 08/18] raise exception instead of warning for ambiguous kwargs

---
 python/nwis_client/src/hydrotools/nwis_client/iv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py
index 5573bb64..ed9cc7e8 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/iv.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py
@@ -35,8 +35,7 @@
 from ._utilities import verify_case_insensitive_kwargs
 
 def _verify_case_insensitive_kwargs_handler(m: str) -> None:
-    warnings.warn("`hydrotools.nwis_client` > 3.1 will raise RuntimeError exception instead of RuntimeWarning.", DeprecationWarning)
-    warnings.warn(m, RuntimeWarning)
+    raise RuntimeError(m)
 
 class IVDataService:
     """

From 9998b0a091a7b8c93aabdf36be26e5b6bde31884 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 16:35:19 -0600
Subject: [PATCH 09/18] use value_time as default

---
 python/nwis_client/src/hydrotools/nwis_client/iv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py
index ed9cc7e8..16ceb0b4 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/iv.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py
@@ -50,7 +50,7 @@ class IVDataService:
         Toggle sqlite3 request caching
     cache_expire_after : int
         Cached item life length in seconds
-    value_time_label: str, default 'value_date'
+    value_time_label: str, default 'value_time'
         Label to use for datetime column returned by IVDataService.get
     cache_filename: str or Path default 'nwisiv_cache'
         Sqlite cache filename or filepath. Suffix '.sqlite' will be added to file if not included.
@@ -105,7 +105,7 @@ class IVDataService:
     def __init__(self, *, 
         enable_cache: bool = True, 
         cache_expire_after: int = 43200,
-        value_time_label: str = None,
+        value_time_label: str = "value_time",
         cache_filename: Union[str, Path] = "nwisiv_cache"
         ):
         self._cache_enabled = enable_cache

From a78a5428e6ff3fb85e5b5f57599a736f83d8ee23 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 16:35:32 -0600
Subject: [PATCH 10/18] update test to reflect new value time column

---
 python/nwis_client/tests/test_nwis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/nwis_client/tests/test_nwis.py b/python/nwis_client/tests/test_nwis.py
index e0b44c1f..374819fb 100644
--- a/python/nwis_client/tests/test_nwis.py
+++ b/python/nwis_client/tests/test_nwis.py
@@ -272,7 +272,7 @@ def test_get_slow(setup_iv):
     df = setup_iv.get(site, startDT=start, endDT=end)
     # IV api seems to send different start based on daylights saving time.
     # Test is less prescriptive, but still should suffice
-    assert df["value_date"][0].isoformat().startswith(start)
+    assert df["value_time"][0].isoformat().startswith(start)
 
 
 datetime_keyword_test_data_should_fail = [

From 0512e4b10052d51dacc623b8d1708e26c4ba76ec Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Fri, 4 Mar 2022 17:41:21 -0600
Subject: [PATCH 11/18] CLiRunner does not like async

---
 python/nwis_client/tests/test_client_cli.py | 27 ++++++++++++---------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/python/nwis_client/tests/test_client_cli.py b/python/nwis_client/tests/test_client_cli.py
index cdabb5ad..ad7c4fd2 100644
--- a/python/nwis_client/tests/test_client_cli.py
+++ b/python/nwis_client/tests/test_client_cli.py
@@ -1,22 +1,25 @@
 import pytest
-from click.testing import CliRunner
-from hydrotools.nwis_client.cli import run
+from hydrotools.nwis_client import cli
 from pathlib import Path
+import subprocess
+from tempfile import TemporaryDirectory
 
 def test_cli():
-    runner = CliRunner()
-
-    with runner.isolated_filesystem():
+    """Normaly would use click.testing.CLiRunner. However, this does not appear to be async friendly."""
+    with TemporaryDirectory() as tdir:
         # Test default parameters
-        result1 = runner.invoke(run, ['01013500', '02146470', "test_output_1.csv"])
-        assert result1.exit_code == 0
-        assert Path("test_output_1.csv").exists()
+        result1 = subprocess.run([
+            "nwis-client", "01013500", "02146470", f"{tdir}/test_output_1.csv"
+        ])
+        assert result1.returncode == 0
+        assert Path(f"{tdir}/test_output_1.csv").exists()
 
         # Test other parameters
-        result2 = runner.invoke(run, [
+        result2 = subprocess.run([
+            "nwis-client",
             '-s', '2022-01-01',
             '-e', '2022-01-02',
             '-p', '00065',
-            '01013500', '02146470', "test_output_2.csv"])
-        assert result2.exit_code == 0
-        assert Path("test_output_2.csv").exists()
+            '01013500', '02146470', f"{tdir}/test_output_2.csv"])
+        assert result2.returncode == 0
+        assert Path(f"{tdir}/test_output_2.csv").exists()

From 39a54ba3338e1b5432f596db2261c2b6ee78a507 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Mon, 7 Mar 2022 13:01:52 -0600
Subject: [PATCH 12/18] add header and comments disable options

---
 .../src/hydrotools/nwis_client/cli.py         | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/cli.py b/python/nwis_client/src/hydrotools/nwis_client/cli.py
index fb0b4eb0..442f1969 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/cli.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/cli.py
@@ -19,10 +19,16 @@ def convert(self, value, param, ctx):
 
 def write_to_csv(
     data: pd.DataFrame,
-    ofile: Path
+    ofile: Path,
+    comments: bool = True,
+    header: bool = True
     ) -> None:
+    # Start building output
+    output = ''
+
     # Comments
-    output = """# USGS IV Service Data
+    if comments:
+        output += """# USGS IV Service Data
 # 
 # value_date: Datetime of measurement (UTC) (character string)
 # variable: USGS variable name (character string)
@@ -33,14 +39,14 @@ def write_to_csv(
 # series: Series number in case multiple time series are returned (integer)
 # 
 """
-    # Add version, link, and write time
-    now = pd.Timestamp.utcnow()
-    output += f"# Generated at {now}\n"
-    output += f"# nwis_client version: {CLIENT_VERSION.__version__}\n"
-    output += "# Source code: https://github.com/NOAA-OWP/hydrotools\n# \n"
+        # Add version, link, and write time
+        now = pd.Timestamp.utcnow()
+        output += f"# Generated at {now}\n"
+        output += f"# nwis_client version: {CLIENT_VERSION.__version__}\n"
+        output += "# Source code: https://github.com/NOAA-OWP/hydrotools\n# \n"
 
     # Write to file
-    output += data.to_csv(index=False, float_format="{:.2f}".format)
+    output += data.to_csv(index=False, float_format="{:.2f}".format, header=header)
     with ofile.open("w") as of:
         of.write(output)
 
@@ -50,12 +56,16 @@ def write_to_csv(
 @click.option("-s", "--startDT", "startDT", nargs=1, type=TimestampParamType(), help="Start datetime")
 @click.option("-e", "--endDT", "endDT", nargs=1, type=TimestampParamType(), help="End datetime")
 @click.option("-p", "--parameterCd", "parameterCd", nargs=1, type=str, default="00060", help="Parameter code")
+@click.option('--comments/--no-comments', default=True, help="Enable/disable comments in output, enabled by default")
+@click.option('--header/--no-header', default=True, help="Enable/disable header in output, enabled by default")
 def run(
     sites: Tuple[str], 
     ofile: Path, 
     startDT: pd.Timestamp = None,
     endDT: pd.Timestamp = None,
-    parameterCd: str = "00060"
+    parameterCd: str = "00060",
+    comments: bool = True,
+    header: bool = True
     ) -> None:
     """Retrieve data from the USGS IV Web Service API and write to CSV.
     Writes data for all SITES to OFILE.
@@ -76,7 +86,7 @@ def run(
     )
     
     # Write to CSV
-    write_to_csv(data=df, ofile=ofile)
+    write_to_csv(data=df, ofile=ofile, comments=comments, header=header)
 
 if __name__ == "__main__":
     run()

From 4ff522d5aa7f91bfb1eb31de621695ccdbecff41 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Mon, 7 Mar 2022 13:02:05 -0600
Subject: [PATCH 13/18] test header and comments disable options

---
 python/nwis_client/tests/test_client_cli.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/python/nwis_client/tests/test_client_cli.py b/python/nwis_client/tests/test_client_cli.py
index ad7c4fd2..6867cba6 100644
--- a/python/nwis_client/tests/test_client_cli.py
+++ b/python/nwis_client/tests/test_client_cli.py
@@ -23,3 +23,22 @@ def test_cli():
             '01013500', '02146470', f"{tdir}/test_output_2.csv"])
         assert result2.returncode == 0
         assert Path(f"{tdir}/test_output_2.csv").exists()
+
+def test_comments_header():
+    """Normaly would use click.testing.CLiRunner. However, this does not appear to be async friendly."""
+    with TemporaryDirectory() as tdir:
+        # Output file
+        ofile = Path(tdir) / "test_output.csv"
+
+        # Disable comments and header
+        result2 = subprocess.run([
+            "nwis-client",
+            '--no-comments', '--no-header',
+            '01013500', '02146470', str(ofile)])
+        assert result2.returncode == 0
+        assert ofile.exists()
+
+        # File should only have two lines
+        with ofile.open('r') as fi:
+            count = len([l for l in fi])
+            assert count == 2
\ No newline at end of file

From 5940cf728a9e02b3f0bcf586e17c1238994b1809 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Wed, 16 Mar 2022 07:46:30 -0500
Subject: [PATCH 14/18] update usage message

---
 python/nwis_client/README.md | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/python/nwis_client/README.md b/python/nwis_client/README.md
index 8831f5a0..fb7b88a5 100644
--- a/python/nwis_client/README.md
+++ b/python/nwis_client/README.md
@@ -73,10 +73,14 @@ Usage: nwis-client [OPTIONS] [SITES]... OFILE
   nwis-client 01013500 02146470 my_output.csv
 
 Options:
-  -s, --startDT TIMESTAMP  Start datetime
-  -e, --endDT TIMESTAMP    End datetime
-  -p, --parameterCd TEXT   Parameter code
-  --help                   Show this message and exit.
+  -s, --startDT TIMESTAMP     Start datetime
+  -e, --endDT TIMESTAMP       End datetime
+  -p, --parameterCd TEXT      Parameter code
+  --comments / --no-comments  Enable/disable comments in output, enabled by
+                              default
+  --header / --no-header      Enable/disable header in output, enabled by
+                              default
+  --help                      Show this message and exit.
 ```
 
 This example retrieves the last discharge value from two sites:

From ebad982ec430512a06579294a547b71c2cf97051 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Wed, 16 Mar 2022 10:02:26 -0500
Subject: [PATCH 15/18] update usage

---
 python/nwis_client/README.md | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/python/nwis_client/README.md b/python/nwis_client/README.md
index fb7b88a5..765bf3d0 100644
--- a/python/nwis_client/README.md
+++ b/python/nwis_client/README.md
@@ -56,23 +56,23 @@ print(observations_data.head())
 ```
 
 ### Command Line Interface (CLI)
-The `hydrotools.nwis_client` package also includes a command-line utility that is installed to the environment targeted by the `pip` installation above. Starting a terminal and activating the correct Python environment should enable the `nwis-client` CLI. This tool will retrieve data from the USGS IV Web Service and write it to CSV format.
+The `hydrotools.nwis_client` package includes a command-line utility.
 
 This example demonstrates calling the help page:
 ```bash
 $ nwis-client --help
 ```
 ```console
-Usage: nwis-client [OPTIONS] [SITES]... OFILE
+Usage: nwis-client [OPTIONS] [SITES]...
 
-  Retrieve data from the USGS IV Web Service API and write to CSV. Writes data
-  for all SITES to OFILE.
+  Retrieve data from the USGS IV Web Service API and write in CSV format.
 
   Example:
 
-  nwis-client 01013500 02146470 my_output.csv
+  nwis-client 01013500 02146470
 
 Options:
+  -o, --output FILENAME       Output file path
   -s, --startDT TIMESTAMP     Start datetime
   -e, --endDT TIMESTAMP       End datetime
   -p, --parameterCd TEXT      Parameter code
@@ -85,8 +85,7 @@ Options:
 
 This example retrieves the last discharge value from two sites:
 ```bash
-$ nwis-client 01013500 02146470 my_data.csv
-$ cat my_data.csv
+$ nwis-client 01013500 02146470
 ```
 ```console
 # USGS IV Service Data
@@ -110,8 +109,7 @@ value_time,variable_name,usgs_site_code,measurement_unit,value,qualifiers,series
 
 This example retrieves stage data from two sites for a specific time period:
 ```bash
-$ nwis-client -p 00065 -s 2021-06-01T00:00 -e 2021-06-01T01:00 01013500 02146470 my_data.csv
-$ cat my_data.csv
+$ nwis-client -p 00065 -s 2021-06-01T00:00 -e 2021-06-01T01:00 01013500 02146470
 ```
 ```console
 # USGS IV Service Data

From dcb9baae44dcb3ae1abcbe1061d5ec3a306baf71 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Wed, 16 Mar 2022 10:08:30 -0500
Subject: [PATCH 16/18] light refactor to adopt input output streams and
 clean-up code

---
 .../src/hydrotools/nwis_client/cli.py         | 58 +++++++++++--------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/cli.py b/python/nwis_client/src/hydrotools/nwis_client/cli.py
index 442f1969..5e39d58f 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/cli.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/cli.py
@@ -2,8 +2,8 @@
 from hydrotools.nwis_client import IVDataService
 from hydrotools.nwis_client import _version as CLIENT_VERSION
 from typing import Tuple
-from pathlib import Path
 import pandas as pd
+from dataclasses import dataclass
 
 class TimestampParamType(click.ParamType):
     name = "timestamp"
@@ -17,18 +17,9 @@ def convert(self, value, param, ctx):
         except ValueError:
             self.fail(f"{value!r} is not a valid timestamp", param, ctx)
 
-def write_to_csv(
-    data: pd.DataFrame,
-    ofile: Path,
-    comments: bool = True,
-    header: bool = True
-    ) -> None:
-    # Start building output
-    output = ''
-
-    # Comments
-    if comments:
-        output += """# USGS IV Service Data
+@dataclass
+class CSVDefaults:
+    comments: str = """# USGS IV Service Data
 # 
 # value_date: Datetime of measurement (UTC) (character string)
 # variable: USGS variable name (character string)
@@ -39,20 +30,35 @@ def write_to_csv(
 # series: Series number in case multiple time series are returned (integer)
 # 
 """
+
+def write_to_csv(
+    data: pd.DataFrame,
+    ofile: click.File,
+    comments: bool = True,
+    header: bool = True
+    ) -> None:
+    # Get default options
+    defaults = CSVDefaults()
+
+    # Comments
+    if comments:
+        output = defaults.comments
+
         # Add version, link, and write time
         now = pd.Timestamp.utcnow()
         output += f"# Generated at {now}\n"
         output += f"# nwis_client version: {CLIENT_VERSION.__version__}\n"
         output += "# Source code: https://github.com/NOAA-OWP/hydrotools\n# \n"
 
-    # Write to file
-    output += data.to_csv(index=False, float_format="{:.2f}".format, header=header)
-    with ofile.open("w") as of:
-        of.write(output)
+        # Write comments to file
+        ofile.write(output)
 
+    # Write data to file
+    data.to_csv(ofile, mode="a", index=False, float_format="{:.2f}".format, header=header, chunksize=2000)
+    
 @click.command()
-@click.argument("sites", nargs=-1)
-@click.argument("ofile", nargs=1, type=click.Path(path_type=Path))
+@click.argument("sites", nargs=-1, required=False)
+@click.option("-o", "--output", nargs=1, type=click.File("w"), help="Output file path", default="-")
 @click.option("-s", "--startDT", "startDT", nargs=1, type=TimestampParamType(), help="Start datetime")
 @click.option("-e", "--endDT", "endDT", nargs=1, type=TimestampParamType(), help="End datetime")
 @click.option("-p", "--parameterCd", "parameterCd", nargs=1, type=str, default="00060", help="Parameter code")
@@ -60,20 +66,24 @@ def write_to_csv(
 @click.option('--header/--no-header', default=True, help="Enable/disable header in output, enabled by default")
 def run(
     sites: Tuple[str], 
-    ofile: Path, 
+    output: click.File, 
     startDT: pd.Timestamp = None,
     endDT: pd.Timestamp = None,
     parameterCd: str = "00060",
     comments: bool = True,
     header: bool = True
     ) -> None:
-    """Retrieve data from the USGS IV Web Service API and write to CSV.
-    Writes data for all SITES to OFILE.
+    """Retrieve data from the USGS IV Web Service API and write in CSV format.
 
     Example:
     
-    nwis-client 01013500 02146470 my_output.csv
+    nwis-client 01013500 02146470
     """
+    # Get sites
+    if not sites:
+        print("Reading sites from stdin: ")
+        sites = click.get_text_stream("stdin").read().split()
+
     # Setup client
     client = IVDataService(value_time_label="value_time")
 
@@ -86,7 +96,7 @@ def run(
     )
     
     # Write to CSV
-    write_to_csv(data=df, ofile=ofile, comments=comments, header=header)
+    write_to_csv(data=df, ofile=output, comments=comments, header=header)
 
 if __name__ == "__main__":
     run()

From c7edd7a2515a7cb87b9099000bb1fd18dc05b280 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Wed, 16 Mar 2022 10:11:45 -0500
Subject: [PATCH 17/18] update for new interface options

---
 python/nwis_client/tests/test_client_cli.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/python/nwis_client/tests/test_client_cli.py b/python/nwis_client/tests/test_client_cli.py
index 6867cba6..5ae3c80a 100644
--- a/python/nwis_client/tests/test_client_cli.py
+++ b/python/nwis_client/tests/test_client_cli.py
@@ -9,7 +9,7 @@ def test_cli():
     with TemporaryDirectory() as tdir:
         # Test default parameters
         result1 = subprocess.run([
-            "nwis-client", "01013500", "02146470", f"{tdir}/test_output_1.csv"
+            "nwis-client", "01013500", "02146470", "-o", f"{tdir}/test_output_1.csv"
         ])
         assert result1.returncode == 0
         assert Path(f"{tdir}/test_output_1.csv").exists()
@@ -20,7 +20,7 @@ def test_cli():
             '-s', '2022-01-01',
             '-e', '2022-01-02',
             '-p', '00065',
-            '01013500', '02146470', f"{tdir}/test_output_2.csv"])
+            '01013500', '02146470', "-o", f"{tdir}/test_output_2.csv"])
         assert result2.returncode == 0
         assert Path(f"{tdir}/test_output_2.csv").exists()
 
@@ -34,11 +34,12 @@ def test_comments_header():
         result2 = subprocess.run([
             "nwis-client",
             '--no-comments', '--no-header',
-            '01013500', '02146470', str(ofile)])
+            '01013500', '02146470', "-o", str(ofile)])
         assert result2.returncode == 0
         assert ofile.exists()
 
         # File should only have two lines
         with ofile.open('r') as fi:
             count = len([l for l in fi])
-            assert count == 2
\ No newline at end of file
+            assert count == 2
+            
\ No newline at end of file

From 93850f8a1d665b642172f3df7022a7aef2db4d19 Mon Sep 17 00:00:00 2001
From: Jason Regina <jarq6c@gmail.com>
Date: Wed, 16 Mar 2022 14:07:31 -0500
Subject: [PATCH 18/18] increase chunk size

---
 python/nwis_client/src/hydrotools/nwis_client/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/nwis_client/src/hydrotools/nwis_client/cli.py b/python/nwis_client/src/hydrotools/nwis_client/cli.py
index 5e39d58f..d8f246e3 100644
--- a/python/nwis_client/src/hydrotools/nwis_client/cli.py
+++ b/python/nwis_client/src/hydrotools/nwis_client/cli.py
@@ -54,7 +54,7 @@ def write_to_csv(
         ofile.write(output)
 
     # Write data to file
-    data.to_csv(ofile, mode="a", index=False, float_format="{:.2f}".format, header=header, chunksize=2000)
+    data.to_csv(ofile, mode="a", index=False, float_format="{:.2f}".format, header=header, chunksize=20000)
     
 @click.command()
 @click.argument("sites", nargs=-1, required=False)