diff --git a/.gitignore b/.gitignore index 0f0a507..d0908c6 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,5 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -.idea/ \ No newline at end of file +.idea/ +test/ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a15bc96..a0084aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,3 +76,5 @@ wcwidth==0.2.5 webencodings==0.5.1 wrapt==1.14.1 yarl==1.8.1 +responses +pytest-cov \ No newline at end of file diff --git a/restDbApi/fetch_flat_data.py b/restDbApi/fetch_flat_data.py deleted file mode 100644 index 302b281..0000000 --- a/restDbApi/fetch_flat_data.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Dict, Any, List - -import requests - - -def get_flat_weather_data(params: Dict[str, Any], url: str) -> List[Dict[str, Any]]: - response = requests.get(url, params=params, ) - payload = response.json() - flat_data = [] - for record in payload["forecast"]["forecastday"]: - data = record["day"] - data['date'] = record['date'] - i = 0 - for hour in record["hour"]: - data["hour_" + str(i) + "_gust_kph"] = hour["gust_kph"] - i += 1 - data["astro"] = record["astro"] - flat_data.append(data) - return flat_data diff --git a/restDbApi/my_weather_adapter.py b/restDbApi/my_weather_adapter.py index 059f5d6..c9ec03c 100644 --- a/restDbApi/my_weather_adapter.py +++ b/restDbApi/my_weather_adapter.py @@ -4,46 +4,16 @@ from shillelagh.adapters.base import Adapter from shillelagh.fields import ( - Boolean, Field, Filter, Float, Integer, - ISODate, - ISODateTime, - ISOTime, - String, ) -from shillelagh.filters import Range, Equal +from shillelagh.filters import Equal from shillelagh.typing import RequestedOrder, Row -from datetime import datetime, date, timedelta import requests -import dateutil.parser -from shillelagh.lib import SimpleCostModel - -from restDbApi.fetch_flat_data import get_flat_weather_data - - -# def get_session() -> requests_cache.CachedSession: -# """ -# Return a cached session. -# """ -# return requests_cache.CachedSession( -# cache_name="generic_json_cache", -# backend="sqlite", -# expire_after=180, -# ) class MyWeatherAdapter(Adapter): - # days = Integer(filters=[Equal]) - # aqi = Boolean(filters=[Equal]) - # alerts = Boolean(filters=[Equal]) - - # maxtemp_c = Float() - # date = ISODate() - # maxwind_kph = Float() - # daily_chance_of_rain = Integer() - # hour_key = String() safe = True supports_limit = True @@ -55,7 +25,6 @@ class MyWeatherAdapter(Adapter): @staticmethod def supports(uri: str, fast: bool = True, **kwargs: Any) -> Optional[bool]: - return False parsed = urllib.parse.urlparse(uri) query_string = urllib.parse.parse_qs(parsed.query) return ( @@ -88,8 +57,6 @@ def get_columns(self) -> Dict[str, Field]: "days": Integer(filters=[Equal]) } - # get_cost = SimpleCostModel(100) - def get_rows( self, bounds: Dict[str, Filter], @@ -103,10 +70,6 @@ def get_rows( url = "http://api.weatherapi.com/v1/forecast.json" params = {"key": self.api_key, "q": self.location, "days": days_param.value, "aqi": aqi_param.value, "alerts": alerts_param.value} - # params = {"key": self.api_key, "q": self.location, "days": 3, "aqi": "no", "alerts": "no"} - # response = get_flat_weather_data(params, url) - - # fetch data from api response = requests.get(url, params=params, ) payload = response.json() flat_data = [] @@ -119,8 +82,7 @@ def get_rows( i += 1 data["astro"] = record["astro"] flat_data.append(data) - - + for i, record in enumerate(flat_data): yield { "rowid": i, diff --git a/restDbApi/rest_api_dialect.py b/restDbApi/rest_api_dialect.py index e7e9dd2..baaf412 100644 --- a/restDbApi/rest_api_dialect.py +++ b/restDbApi/rest_api_dialect.py @@ -37,4 +37,5 @@ def get_table_names(self, connection, schema=None, **kw) -> List[str]: return ["'Tables' dont exists in rest APIs. Use SQL lab directly"] def do_ping(self, dbapi_connection: _ConnectionFairy) -> bool: + # required to check 'active' connections by superset. As this is a REST call, this is not applicable. return True diff --git a/restDbApi/rest_api_engine_spec.py b/restDbApi/rest_api_engine_spec.py index 8bdedc4..6e30a86 100644 --- a/restDbApi/rest_api_engine_spec.py +++ b/restDbApi/rest_api_engine_spec.py @@ -1,10 +1,9 @@ -# Taken from: https://github.com/apache/superset/blob/master/superset/db_engine_specs/gsheets.py # noqa: E501 +""" Optional use in superset from superset.db_engine_specs.sqlite import SqliteEngineSpec class RestApiEngineSpec(SqliteEngineSpec): - """Engine for REST API""" - + # Engine for REST API engine = "rest" engine_name = "REST" allows_joins = True @@ -12,6 +11,5 @@ class RestApiEngineSpec(SqliteEngineSpec): default_driver = "apsw" sqlalchemy_uri_placeholder = "rest://" - - # TODO(cancan101): figure out what other spec items make sense here - # See: https://preset.io/blog/building-database-connector/ + # https://preset.io/blog/building-database-connector/ +""" diff --git a/test/datasette_cache.sqlite b/test/datasette_cache.sqlite deleted file mode 100644 index 30a9394..0000000 Binary files a/test/datasette_cache.sqlite and /dev/null differ diff --git a/test/generic_json_cache.sqlite b/test/generic_json_cache.sqlite deleted file mode 100644 index b11f9b3..0000000 Binary files a/test/generic_json_cache.sqlite and /dev/null differ diff --git a/test/rest_api_cache.sqlite b/test/rest_api_cache.sqlite index f704312..3aed13e 100644 Binary files a/test/rest_api_cache.sqlite and b/test/rest_api_cache.sqlite differ diff --git a/test/testCorrectAdapter.py b/test/testCorrectAdapter.py deleted file mode 100644 index 0537699..0000000 --- a/test/testCorrectAdapter.py +++ /dev/null @@ -1,18 +0,0 @@ -from datetime import datetime, timedelta -from shillelagh.backends.apsw.db import connect - -three_days_ago = datetime.now() - timedelta(days=3) - -# sign up for an API key at https://www.weatherapi.com/my/ -api_key = "349be02b64f54bbd9bb70758232302" - -connection = connect(":memory:", adapter_kwargs={"weatherapi": {"api_key": api_key}}) -cursor = connection.cursor() - -sql = """ -SELECT * -FROM "https://api.weatherapi.com/v1/history.json?q=London" -WHERE time >= ? -""" -for row in cursor.execute(sql, (three_days_ago,)): - print(row) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3c4d944 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,33 @@ +from typing import Generator + +import pytest +from sqlalchemy.engine import Connection, Engine +from sqlalchemy import create_engine +import responses +from shillelagh.backends.apsw.db import connect + +GET_SQL_ALCHEMY_URI = 'rest://api.covidtracking.com?ishttps=1' +POST_SQL_ALCHEMY_URI = 'rest://some.api.com?ishttps=0' + +@pytest.fixture +def simple_url() -> str: + return GET_SQL_ALCHEMY_URI + + +@pytest.fixture +def post_sql_uri(): + return POST_SQL_ALCHEMY_URI + + +@pytest.fixture +def covid_data_connection(simple_url: str) -> Generator[Connection, None, None]: + engine = create_engine(simple_url) + with engine.connect() as connection: + yield connection + + +@pytest.fixture +def post_data_connection(post_sql_uri: str) -> Generator[Connection, None, None]: + engine = create_engine(post_sql_uri) + with engine.connect() as connection: + yield connection diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..a0ad436 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,159 @@ +import pytest +from pytest_mock import MockerFixture +from requests import Session +from requests_mock.mocker import Mocker +import urllib +import restDbApi.rest_api_dialect +from sqlalchemy import inspect + +SIMPLE_URL = 'https://api.covidtracking.com/v1/us/daily.json' +POST_URL = 'http://some.api.com/some/api/path?a=60=-50&c=someQuery' + +def test_rest_adapter(mocker: MockerFixture, requests_mock: Mocker, covid_data_connection): + mocker.patch( + "restDbApi.rest_api_adapter.requests_cache.CachedSession", + return_value=Session(), + ) + + requests_mock.get(SIMPLE_URL, json= [ + { + "date": 20210307, + "states": 56, + "positive": 28756489, + "negative": 74582825, + "pending": 11808, + "hospitalizedCurrently": 40199, + "hospitalizedCumulative": 776361, + "inIcuCurrently": 8134, + "inIcuCumulative": 45475, + "onVentilatorCurrently": 2802, + "onVentilatorCumulative": 4281, + "dateChecked": "2021-03-07T24:00:00Z", + "death": 515151, + "hospitalized": 776361, + "totalTestResults": 363825123, + "lastModified": "2021-03-07T24:00:00Z", + "recovered": None, + "total": 0, + "posNeg": 0, + "deathIncrease": 842, + "hospitalizedIncrease": 726, + "negativeIncrease": 131835, + "positiveIncrease": 41835, + "totalTestResultsIncrease": 1170059, + "hash": "a80d0063822e251249fd9a44730c49cb23defd83" + }, + { + "date": 20210306, + "states": 56, + "positive": 28714654, + "negative": 74450990, + "pending": 11783, + "hospitalizedCurrently": 41401, + "hospitalizedCumulative": 775635, + "inIcuCurrently": 8409, + "inIcuCumulative": 45453, + "onVentilatorCurrently": 2811, + "onVentilatorCumulative": 4280, + "dateChecked": "2021-03-06T24:00:00Z", + "death": 514309, + "hospitalized": 775635, + "totalTestResults": 362655064, + "lastModified": "2021-03-06T24:00:00Z", + "recovered": None, + "total": 0, + "posNeg": 0, + "deathIncrease": 1680, + "hospitalizedIncrease": 503, + "negativeIncrease": 143835, + "positiveIncrease": 60015, + "totalTestResultsIncrease": 1430992, + "hash": "dae5e558c24adb86686bbd58c08cce5f610b8bb0" + }]) + + sql = "select * from '/v1/us/daily.json'" + data = covid_data_connection.execute(sql) + assert len(list(data)) == 2 + +def test_rest_adapter_post(mocker: MockerFixture, requests_mock: Mocker, post_data_connection): + mocker.patch( + "restDbApi.rest_api_adapter.requests_cache.CachedSession", + return_value=Session(), + ) + + endpoint = '/some/api/path?a=60=-50&c=someQuery' + headers = '&header1=Content-Type:application/json' + headers += '&header2=IAM_ID:satvik' + headers += '&header3=ENVIRONMENT:staging:1.5.3' + headers += '&header4=NAME:MY-REST-SERVICE' + body = '{ "name": "satvik", "interests": [ { "name": "badminton", "category": "sports", "stats": { "racket": "intermediate", "shuttle": "yonex mavis 500" } }, { "name": "programming", "category": "computers", "stats": { "laptop": "mac book pro", "mouse": "5D ergonomic", "keyboard": "broken" } } ] }' + jsonpath = "#$[*]" + + + encoded_body = "&body=" + urllib.parse.quote(body, 'utf8') + url = endpoint + headers + encoded_body + jsonpath + sql = f"select * from '{url}'" + + requests_mock.post(POST_URL, json= [ + { + "date": 20210307, + "states": 56, + "positive": 28756489, + "negative": 74582825, + "pending": 11808, + "hospitalizedCurrently": 40199, + "hospitalizedCumulative": 776361, + "inIcuCurrently": 8134, + "inIcuCumulative": 45475, + "onVentilatorCurrently": 2802, + "onVentilatorCumulative": 4281, + "dateChecked": "2021-03-07T24:00:00Z", + "death": 515151, + "hospitalized": 776361, + "totalTestResults": 363825123, + "lastModified": "2021-03-07T24:00:00Z", + "recovered": None, + "total": 0, + "posNeg": 0, + "deathIncrease": 842, + "hospitalizedIncrease": 726, + "negativeIncrease": 131835, + "positiveIncrease": 41835, + "totalTestResultsIncrease": 1170059, + "hash": "a80d0063822e251249fd9a44730c49cb23defd83" + }, + { + "date": 20210306, + "states": 56, + "positive": 28714654, + "negative": 74450990, + "pending": 11783, + "hospitalizedCurrently": 41401, + "hospitalizedCumulative": 775635, + "inIcuCurrently": 8409, + "inIcuCumulative": 45453, + "onVentilatorCurrently": 2811, + "onVentilatorCumulative": 4280, + "dateChecked": "2021-03-06T24:00:00Z", + "death": 514309, + "hospitalized": 775635, + "totalTestResults": 362655064, + "lastModified": "2021-03-06T24:00:00Z", + "recovered": None, + "total": 0, + "posNeg": 0, + "deathIncrease": 1680, + "hospitalizedIncrease": 503, + "negativeIncrease": 143835, + "positiveIncrease": 60015, + "totalTestResultsIncrease": 1430992, + "hash": "dae5e558c24adb86686bbd58c08cce5f610b8bb0" + }]) + data = post_data_connection.execute(sql) + assert len(list(data)) == 2 + +def test_dialect_get_table_name(covid_data_connection): + metadata = inspect(covid_data_connection) + tables = metadata.get_table_names() + assert len(tables) == 1 + assert tables[0] == "'Tables' dont exists in rest APIs. Use SQL lab directly" \ No newline at end of file