add tests + code refactor

.gitignore

@@ -157,4 +157,5 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-.idea/
+.idea/
+test/

requirements.txt

@@ -76,3 +76,5 @@ wcwidth==0.2.5
 webencodings==0.5.1
 wrapt==1.14.1
 yarl==1.8.1
+responses
+pytest-cov

restDbApi/my_weather_adapter.py

@@ -4,46 +4,16 @@
 
 from shillelagh.adapters.base import Adapter
 from shillelagh.fields import (
-    Boolean,
     Field,
     Filter,
     Float,
     Integer,
-    ISODate,
-    ISODateTime,
-    ISOTime,
-    String,
 )
-from shillelagh.filters import Range, Equal
+from shillelagh.filters import Equal
 from shillelagh.typing import RequestedOrder, Row
-from datetime import datetime, date, timedelta
 import requests
-import dateutil.parser
-from shillelagh.lib import SimpleCostModel
-
-from restDbApi.fetch_flat_data import get_flat_weather_data
-
-
-# def get_session() -> requests_cache.CachedSession:
-#     """
-#     Return a cached session.
-#     """
-#     return requests_cache.CachedSession(
-#         cache_name="generic_json_cache",
-#         backend="sqlite",
-#         expire_after=180,
-#     )
 
 class MyWeatherAdapter(Adapter):
-    # days = Integer(filters=[Equal])
-    # aqi = Boolean(filters=[Equal])
-    # alerts = Boolean(filters=[Equal])
-
-    # maxtemp_c = Float()
-    # date = ISODate()
-    # maxwind_kph = Float()
-    # daily_chance_of_rain = Integer()
-    # hour_key = String()
 
     safe = True
     supports_limit = True
@@ -55,7 +25,6 @@ class MyWeatherAdapter(Adapter):
 
     @staticmethod
     def supports(uri: str, fast: bool = True, **kwargs: Any) -> Optional[bool]:
-        return False
         parsed = urllib.parse.urlparse(uri)
         query_string = urllib.parse.parse_qs(parsed.query)
         return (
@@ -88,8 +57,6 @@ def get_columns(self) -> Dict[str, Field]:
             "days": Integer(filters=[Equal])
         }
 
-    # get_cost = SimpleCostModel(100)
-
     def get_rows(
             self,
             bounds: Dict[str, Filter],
@@ -103,10 +70,6 @@ def get_rows(
 
         url = "http://api.weatherapi.com/v1/forecast.json"
         params = {"key": self.api_key, "q": self.location, "days": days_param.value, "aqi": aqi_param.value, "alerts": alerts_param.value}
-        # params = {"key": self.api_key, "q": self.location, "days": 3, "aqi": "no", "alerts": "no"}
-        # response = get_flat_weather_data(params, url)
-
-        # fetch data from api
         response = requests.get(url, params=params, )
         payload = response.json()
         flat_data = []
@@ -119,8 +82,7 @@ def get_rows(
                 i += 1
             data["astro"] = record["astro"]
             flat_data.append(data)
-
-
+
         for i, record in enumerate(flat_data):
             yield {
                 "rowid": i,

restDbApi/rest_api_dialect.py

@@ -37,4 +37,5 @@ def get_table_names(self, connection, schema=None, **kw) -> List[str]:
         return ["'Tables' dont exists in rest APIs. Use SQL lab directly"]
 
     def do_ping(self, dbapi_connection: _ConnectionFairy) -> bool:
+        # required to check 'active' connections by superset. As this is a REST call, this is not applicable.
         return True

restDbApi/rest_api_engine_spec.py

@@ -1,17 +1,15 @@
-# Taken from: https://github.com/apache/superset/blob/master/superset/db_engine_specs/gsheets.py  # noqa: E501
+""" Optional use in superset
 from superset.db_engine_specs.sqlite import SqliteEngineSpec
 
 
 class RestApiEngineSpec(SqliteEngineSpec):
-    """Engine for REST API"""
-
+    # Engine for REST API
     engine = "rest"
     engine_name = "REST"
     allows_joins = True
     allows_subqueries = True
 
     default_driver = "apsw"
     sqlalchemy_uri_placeholder = "rest://"
-
-    # TODO(cancan101): figure out what other spec items make sense here
-    # See: https://preset.io/blog/building-database-connector/
+    # https://preset.io/blog/building-database-connector/
+"""

tests/conftest.py

@@ -0,0 +1,33 @@
+from typing import Generator
+
+import pytest
+from sqlalchemy.engine import Connection, Engine
+from sqlalchemy import create_engine
+import responses
+from shillelagh.backends.apsw.db import connect
+
+GET_SQL_ALCHEMY_URI = 'rest://api.covidtracking.com?ishttps=1'
+POST_SQL_ALCHEMY_URI = 'rest://some.api.com?ishttps=0'
+
+@pytest.fixture
+def simple_url() -> str:
+    return GET_SQL_ALCHEMY_URI
+
+
+@pytest.fixture
+def post_sql_uri():
+    return POST_SQL_ALCHEMY_URI
+
+
+@pytest.fixture
+def covid_data_connection(simple_url: str) -> Generator[Connection, None, None]:
+    engine = create_engine(simple_url)
+    with engine.connect() as connection:
+        yield connection
+
+
+@pytest.fixture
+def post_data_connection(post_sql_uri: str) -> Generator[Connection, None, None]:
+    engine = create_engine(post_sql_uri)
+    with engine.connect() as connection:
+        yield connection

tests/tests.py

@@ -0,0 +1,159 @@
+import pytest
+from pytest_mock import MockerFixture
+from requests import Session
+from requests_mock.mocker import Mocker
+import urllib
+import restDbApi.rest_api_dialect
+from sqlalchemy import inspect
+
+SIMPLE_URL = 'https://api.covidtracking.com/v1/us/daily.json'
+POST_URL = 'http://some.api.com/some/api/path?a=60=-50&c=someQuery'
+
+def test_rest_adapter(mocker: MockerFixture, requests_mock: Mocker, covid_data_connection):
+    mocker.patch(
+        "restDbApi.rest_api_adapter.requests_cache.CachedSession",
+        return_value=Session(),
+    )
+
+    requests_mock.get(SIMPLE_URL, json= [
+      {
+    "date": 20210307,
+    "states": 56,
+    "positive": 28756489,
+    "negative": 74582825,
+    "pending": 11808,
+    "hospitalizedCurrently": 40199,
+    "hospitalizedCumulative": 776361,
+    "inIcuCurrently": 8134,
+    "inIcuCumulative": 45475,
+    "onVentilatorCurrently": 2802,
+    "onVentilatorCumulative": 4281,
+    "dateChecked": "2021-03-07T24:00:00Z",
+    "death": 515151,
+    "hospitalized": 776361,
+    "totalTestResults": 363825123,
+    "lastModified": "2021-03-07T24:00:00Z",
+    "recovered": None,
+    "total": 0,
+    "posNeg": 0,
+    "deathIncrease": 842,
+    "hospitalizedIncrease": 726,
+    "negativeIncrease": 131835,
+    "positiveIncrease": 41835,
+    "totalTestResultsIncrease": 1170059,
+    "hash": "a80d0063822e251249fd9a44730c49cb23defd83"
+  },
+  {
+    "date": 20210306,
+    "states": 56,
+    "positive": 28714654,
+    "negative": 74450990,
+    "pending": 11783,
+    "hospitalizedCurrently": 41401,
+    "hospitalizedCumulative": 775635,
+    "inIcuCurrently": 8409,
+    "inIcuCumulative": 45453,
+    "onVentilatorCurrently": 2811,
+    "onVentilatorCumulative": 4280,
+    "dateChecked": "2021-03-06T24:00:00Z",
+    "death": 514309,
+    "hospitalized": 775635,
+    "totalTestResults": 362655064,
+    "lastModified": "2021-03-06T24:00:00Z",
+    "recovered": None,
+    "total": 0,
+    "posNeg": 0,
+    "deathIncrease": 1680,
+    "hospitalizedIncrease": 503,
+    "negativeIncrease": 143835,
+    "positiveIncrease": 60015,
+    "totalTestResultsIncrease": 1430992,
+    "hash": "dae5e558c24adb86686bbd58c08cce5f610b8bb0"
+  }])
+
+    sql = "select * from '/v1/us/daily.json'"
+    data = covid_data_connection.execute(sql)
+    assert len(list(data)) == 2
+
+def test_rest_adapter_post(mocker: MockerFixture, requests_mock: Mocker, post_data_connection):
+    mocker.patch(
+        "restDbApi.rest_api_adapter.requests_cache.CachedSession",
+        return_value=Session(),
+    )
+
+    endpoint = '/some/api/path?a=60=-50&c=someQuery'
+    headers = '&header1=Content-Type:application/json'
+    headers += '&header2=IAM_ID:satvik'
+    headers += '&header3=ENVIRONMENT:staging:1.5.3'
+    headers += '&header4=NAME:MY-REST-SERVICE'
+    body = '{ "name": "satvik", "interests": [ { "name": "badminton", "category": "sports", "stats": { "racket": "intermediate", "shuttle": "yonex mavis 500" } }, { "name": "programming", "category": "computers", "stats": { "laptop": "mac book pro", "mouse": "5D ergonomic", "keyboard": "broken" } } ] }'
+    jsonpath = "#$[*]"
+
+
+    encoded_body = "&body=" + urllib.parse.quote(body, 'utf8')
+    url = endpoint + headers + encoded_body + jsonpath
+    sql = f"select * from '{url}'"
+
+    requests_mock.post(POST_URL, json= [
+      {
+    "date": 20210307,
+    "states": 56,
+    "positive": 28756489,
+    "negative": 74582825,
+    "pending": 11808,
+    "hospitalizedCurrently": 40199,
+    "hospitalizedCumulative": 776361,
+    "inIcuCurrently": 8134,
+    "inIcuCumulative": 45475,
+    "onVentilatorCurrently": 2802,
+    "onVentilatorCumulative": 4281,
+    "dateChecked": "2021-03-07T24:00:00Z",
+    "death": 515151,
+    "hospitalized": 776361,
+    "totalTestResults": 363825123,
+    "lastModified": "2021-03-07T24:00:00Z",
+    "recovered": None,
+    "total": 0,
+    "posNeg": 0,
+    "deathIncrease": 842,
+    "hospitalizedIncrease": 726,
+    "negativeIncrease": 131835,
+    "positiveIncrease": 41835,
+    "totalTestResultsIncrease": 1170059,
+    "hash": "a80d0063822e251249fd9a44730c49cb23defd83"
+  },
+  {
+    "date": 20210306,
+    "states": 56,
+    "positive": 28714654,
+    "negative": 74450990,
+    "pending": 11783,
+    "hospitalizedCurrently": 41401,
+    "hospitalizedCumulative": 775635,
+    "inIcuCurrently": 8409,
+    "inIcuCumulative": 45453,
+    "onVentilatorCurrently": 2811,
+    "onVentilatorCumulative": 4280,
+    "dateChecked": "2021-03-06T24:00:00Z",
+    "death": 514309,
+    "hospitalized": 775635,
+    "totalTestResults": 362655064,
+    "lastModified": "2021-03-06T24:00:00Z",
+    "recovered": None,
+    "total": 0,
+    "posNeg": 0,
+    "deathIncrease": 1680,
+    "hospitalizedIncrease": 503,
+    "negativeIncrease": 143835,
+    "positiveIncrease": 60015,
+    "totalTestResultsIncrease": 1430992,
+    "hash": "dae5e558c24adb86686bbd58c08cce5f610b8bb0"
+  }])
+    data = post_data_connection.execute(sql)
+    assert len(list(data)) == 2
+
+def test_dialect_get_table_name(covid_data_connection):
+    metadata = inspect(covid_data_connection)
+    tables = metadata.get_table_names()
+    assert len(tables) == 1
+    assert tables[0] == "'Tables' dont exists in rest APIs. Use SQL lab directly"