From 485ab92653a0aba5080701d53f820df6c563336e Mon Sep 17 00:00:00 2001 From: Satvik Date: Sat, 25 Feb 2023 15:01:18 +0530 Subject: [PATCH] v2 with headers and post body --- .gitignore | 161 +++++++++++++++++- .../rest_api_adapter.cpython-38.pyc | Bin 3847 -> 5583 bytes restDbApi/rest_api_adapter.py | 128 +++++++++----- restDbApi/rest_api_dialect.py | 40 +++++ restDbApi/rest_api_engine_spec.py | 17 ++ setup.py | 1 + test/rest_api_cache.sqlite | Bin 167936 -> 270336 bytes test/testDialect.py | 43 +++++ 8 files changed, 346 insertions(+), 44 deletions(-) create mode 100644 restDbApi/rest_api_dialect.py create mode 100644 restDbApi/rest_api_engine_spec.py create mode 100644 test/testDialect.py diff --git a/.gitignore b/.gitignore index 250170a..0f0a507 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,160 @@ -.idea -*.egg-info +# Byte-compiled / optimized / DLL files __pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: + .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ \ No newline at end of file diff --git a/restDbApi/__pycache__/rest_api_adapter.cpython-38.pyc b/restDbApi/__pycache__/rest_api_adapter.cpython-38.pyc index a100b771ea630670b6c8b49566408f2af9ccbc00..5c28a9e1c3a51d2f700a23e9c5e99b7a8631317a 100644 GIT binary patch literal 5583 zcmb7INpBp-74GV7dKL~>iKI5uwrpn{nNpn1Fcd|y7R!lDDkSA39UGnIR1Miw&%&yn zNFtg*0MeEnz`6LU19^+?Ip>GeH7EZB*$9yDRS(G#B_l`=>P;y6c9oi}S)0j&NG`ti{Kk*nFH%p#Kf@pXZZkPw^9IzxkoXPx9$qi%;*^;ah0U z@L9BGMR~^wf5h3c)j0Jn$!ekD>tL~$CEZS(YUf(#k@i;ly;NxTM$*i*dnZwu4nE3+ zjI*w6FkQOF;~ts~N0&b6c2f~|w0A2JDcAn3B*n;6roEe+{Wxvdy1pbf`$A=cFG?=gWFd2!)xoULnjJe%5*q8~IXXec5T!S%x1Ejq50C3?#EQ7-!t(4tKwHaCUF&waOfxB{O+0o@>U< zb;0M9PzsWmH-lkag7GNsC6O7{zS>NaOnm!KEZK0vis_9yaa)9?)wp?Iba+@34||Ce zQQRUC;ry~VzZ$GDa^38-Tx|5zl4r4)A>UoQcuxtbE~-nNOYd&BFZN{I%n~Io5*zRE z)px*&soz+=HmDiq;zA$uy+^vbCbDR7h#5M4;UF%v>GvP`3UO}hanWf)>O9&|-A=UH z<&Px2-GrMQJ~nFPKt_l(NJk@*k0T+-bJTx=jxiWJ;_-D1DPqZDbtW75 z$5BRmX`FP9UV8B$bg09F)X^9|$g(A0rG;Gi8vZ10p*_R7e1rO#e4F|Yl{CcP3o?8Q zZz>X-#pDIhCRK+YvdOC`ih~F%WZVjEvddj7bcD+t__@nH_^&r;`MiXdFG?cd)~qGU z!WFLZ{Q$lW)Yx@*SXjZh3oNOkuf|7AUzOKk=cBrE8$NSe#PGeQgKZBUsuQo*QH+n| zEVm*nhk_&Ubs!g8JNE@3;V*I87w}tGiL@o>F-cw|a!fEm6eXP`i=rRXvW2dEwl#Ug z#Qeb5lf4)giK&4Owx))5J$&K++0h*H z-IwfFqU9hL8avo+;@EB&hIy9Ojp)^jW=RS-1SvAU#M*`mZdjj9ZQHrMXYE^1iM_}6 zt;^PCki*joQ?=z!iKU-`G#vSJ{DswGmxvQ$`$glpjB4b0(SRMAr2-Jkc3HqC*w*wA z`@?VI47I`P@a(w#;z3BuhyFzp$*n!aT5c0emG9v%9B(BOc--u!0Ig#92Mux3z|i2hk>Olj#v+45 z*I12>v8{7MhknU~g+@)oq6y%mNLQk$-R1q1>a{4M_*RU_pI~9_s7xMdM%PppXGyay zvh^;PWX+PaEq_UbI2qE9aUudfb6f%(MK~3(z^yxV$7bhzo0+jDx}SRtzN1H-f!vND zC3H9oT@+9Fe2U-$&`}Wqk_P(;fQGxf9)P1N2wG|=BPexInzaaAOsiyS#YmazEf2Fj zK#Tz;%pT<@{>RL+Ff;Mim?-n|li;z{V!XnuyCdN^4|olLH^S?HxCyi8q=7c-9pz)_ zof0R+N$fPuPnz)=!&BXGJWF-(S`nuXX<3{WXTat76Q57;$z3lz`xv$vo`Z=^>*^BX z=pcDnX}H?y%S5}a82OEV|3NI*23~3)d?01=QnJY3l8nj9P3mMXtQhsd{|Kz)4I(#* z9LGb|@AbMeQy*inKni17DI0#?+4DQ7?oNe3mxTt&R zb4<%etF+I0mpI#Y*Q{+1=;EW4a%V5tCzQGUsl6R+m-Aq+3|D2cmsR$vd6_%27FMa` z-d-)Q>@#NN0eA1)@{4Sw7~NsprOjJ;IrsAb$aD+y>N)LPg**FsHFxt;%cCS~Vaq-L zr?)?ozaYtw6tzpaMq^YG-W9<%K;B?nPrE(J^n$_z$NHT zP!#!{6;zmi_zR#Y-*)Xfd){MzIDsL2AXg>OP@jTifQuct5F(wmXD8O0wQs|H;I{Gt z`Vb>$tvzSo+F>0VPHJavhNOB2zfYZQ=PTtnAPKbi**W6tOO#pgy~e(T zb^)b0!Pm|=_NEKX+PCP=*D`lJ_-EH#=sfWd*Az)Gg|KbUU((e=RVHpewPl16?PZU8 zLLCWSS1wbs((JZ-S3V~;DB$_N{k?7Za1kGum8}cIZZT{F^Ow_ZGfvf&`J+V|C0spZ z0)%i=b7M7Eh&YS2i%d+(apJ-Sct~A3FeTTRl6GbHfzq`hu9ap3BWYrq$@yi7+Fwx& zrxu)djLjf%s`>VJTQfhz>2qQy1f)VZuU^b=@1tA3EZa1fx&hCBr@1CTA!Qrhm5zIG zF5JdeyZsJV1KFoZJ3)-=wpwtcB2JkwL$-SGkp74#;zIomgpzN09+YhWD>i@*8$gQ< z;KK&lguCr`W>9AvKIq{nx4(9_{oLOx?IYM^gdk;De1I@h$q|sY%Nc^q4nqJsj{sCb z(5U29?p(IA+TKWBrGS&w^Xd-!1hU%vv$1q{0d_|a4%9BNf{YdsQ{k^bZ4Ww3I3kMy z>_4C%@dO_iDa)n5l2w|pi2T1`&c8QFE=#$qzxw&-h!<*5< zy^lY*xfCtlik23CzpR78IH3)SM3P)xJ8)H5qUIbZcgo?1V@WKyJ zyMdyvf&?acExsxDBIE@eh`L&kh~ shta|1;+k}Yu+D^+fX&%8Hs#TsYYIkPLkVosUWK{NSarr7bLyx53-D132mk;8 literal 3847 zcmZ`+OLG*-5$@{O^gJ|r3kd{hS(asZNG#z9hiy!(77%95N`xM3M=!XeHB}?E=$;u> z^#m)$=p=>NJ#g}sgLLG+SN{ng>*~|~gk{4K{<3BU1P=E^W!I~#vcAmxvSwpwsAS=p zs{T^{XT`GqO^wr!i^i8I>H@PYZgG}cA-(OCWp-#YsynHZxuKhRp_logpB2JF7KA}o z42xMQESWiOTFxqA#nioYD6585`aT;DhqIAzBpVG!v$1e28xO~`iEtvD3@6Q8Kb^|X zglDqpaN4vB=}dMuJe!>h&zW|R&StaWY<508pIr!{q|xzWc+tG)!cXunrkAo$!%wrz z;pObJ@U#A$E8!K4mD10%tKrptUoEWhGOujg;Wa)KUKh3F8{*~(3+Kg^odsTf&G;}M zIda2W4=p~*$BryMCMt(c_yuQcR&D%eLT(X179;j30F6>(GASXui^Ns4|fpSI#$ z7alcpv4wWw(b~%WI2XFyudFATFukVYujMjnZq;n)W5hu4w}hq%{L`8O?e>Oze3?d@a3MjGe2XyORL z*J~@wPvU%AyW4r*)*eZf9RHnJZ7tht>`nKS6N+<<5=gnYPl{ilnx0A>WYhUfANiKf=2}?rzlIe|_F#)t0 z*Pn|f*A?-7JCP!a8-zMwkv&*ce~qDgW~a5@MsF-x!(v9fU0m2uLaGIIt9k48Znn^t zaXn9zSRgfS^39vj#MJL^F7|6ixwvQI`SzYJZ;3qWAL1gN-bAG9GMj$?k=N1pKTrke zq(}V=LRcZ=R%mk;Iw&rRCwx9B3L+3iQ4+xnBKEaQP#$@QEG!EjV-=J_!^W8R+U7nl z9Jv@9;#1Vmi?8jMR)g^pFCPtsRbJsk@c$|whW`(X5zHAi-%{@gA4Tt&7#9;*VUnLQ z<5M&)&X|3ssg71j%)FCDaaNqe?qjcgKF%kOym0oVqEJbUV$xGZ? zvvTjyI&i!0v2SL1-0pfi1@6pP&wc3OE-&qm#O-#P)MJX|g&oyu?pNw9gkCehneVkl zEzo{PrfIUNy>={>&;gpgny=uZaJBKZjVDi5R@WcgkDjcqtgqZ#dAw!}i}(;R*9DUB zqna;oLTx!uVgW+?W+BYzdx#F;S?yL6UiZ!mN0duQs zb?t_2aD3}t{_ff{Z4o;(WW1FDzgj`w!n*Pc5~Q{+beg-J7Q*n4)G7hUPO}ecBt<)o z>!RlL^!+W(aN8i6kC7;vFCsY1BIYO)D|W>mV-xoNOkf~C0o2>+oBwapG zh~Or>jiab55DB1tc7YW50c8dXZm}N#KllTxzHPp6GHeO1LL0dlSwA&dpQs-xHrDQQv8UuY(!xc$LQWs}fGhuG zdS&Z{(jXr445PKutnUI(Fu`zue2CgQiqX_!0UKe{_LyDq?YH~W|4-8ofG0~(xksxI zpLVSs*i2Fz2>!MAB9%UlI+MrxT62sm!^~{|%;|mUz=MbGk4j$Z5ks>VXIf^1jCH0QsT01uQ#_#&%lm~^}MGrFFLC_7r>;;S!^Fp_H z$nIOuE|30F_qK6Vds%zHJ>j*oT*7ABf&L~M zwM&UWo=2x#BtaCWi=x@d=pKj5MU=L-=yK+9vDw*@vly>cj4kCe`hYx6{*6RLg2-O} z4nh~&@gDAXyifhoT*6|_674{jk5HEz_#{E;5q?mG%}e-=nD>YAKXA4hTZbikjJLd8 zyFk1|7osTAr6|f;ypvMB5=C@P?2ouAZU|i+B#o$Ntl@X7eIGSf4yX5>?1a`oKYS$s#qUNDw@Fs85-7k){LMG~D@!rWgXfff{ugdJlRz z)3@1d)n;ddsZw=h$_acp#;-OqY89c6;Aj3X(tLN@WHDW)NH*LI5}|WwAB2LNckkQ= z0dsB&tKMmi-lVObR@+IMiZtHZp6_1{)%<3RG+(0O(Nn_>`g5UVnN~5UAEFv@lN=^N zw`Vy@Vhlo$eefy$fl!iY)v)0Q3-`4p9DC@Uq1rTw83?+1n7<2357YmGPM)OmCWRuk z4LaZ=?+=1IkujqoHWLE!bF&ZykICE=K)OJoLOA_~w#uoi2>U)Z0VFBZB@v3eO5!>R wg8ft^7-Nzhb#SNWzjukl)Mbc(RoG>_!fupD5ET=Mi@?YV3v4oU$sQ^F7q1M(m;e9( diff --git a/restDbApi/rest_api_adapter.py b/restDbApi/rest_api_adapter.py index fa31a67..de6e32b 100644 --- a/restDbApi/rest_api_adapter.py +++ b/restDbApi/rest_api_adapter.py @@ -1,32 +1,25 @@ import urllib -from dataclasses import Field -from typing import Optional, Any, Tuple, Union, Dict, List, Iterator +from typing import Optional, Any, Tuple, Dict, List, Iterator from shillelagh.adapters.base import Adapter from shillelagh.fields import ( Boolean, Field, Filter, - Float, - Integer, - ISODate, - ISODateTime, - ISOTime, - String, ) -from shillelagh.filters import Range, Equal +from shillelagh.filters import Equal from shillelagh.typing import RequestedOrder, Row -from datetime import datetime, date, timedelta -import requests -import dateutil.parser from shillelagh.lib import SimpleCostModel, analyze, flatten import requests_cache from jsonpath import JSONPath import logging +import json SUPPORTED_PROTOCOLS = {"http", "https"} AVERAGE_NUMBER_OF_ROWS = 100000 _logger = logging.getLogger(__name__) +CHARSET = 'utf8' + def get_session() -> requests_cache.CachedSession: """ @@ -38,6 +31,40 @@ def get_session() -> requests_cache.CachedSession: expire_after=180, ) + +def get_decoded_json_body(encoded_json_body: str) -> Dict[Any, Any]: + decoded = urllib.parse.unquote(encoded_json_body, CHARSET) + return json.loads(decoded) + +def get_encoded_json_body(plain_json_body: str) -> str: + return urllib.parse.quote(plain_json_body, CHARSET) + +class HttpHeader: + def __init__(self, key, value): + self.key = key + self.value = value + + def get_value(self) -> str: + return self.value + + def get_key(self) -> str: + return self.key + + @staticmethod + def load_headers(headers: List['HttpHeader']) -> Dict[str, str]: + header_dict = {} + for header in headers: + header_dict[header.get_key()] = header.get_value() + return header_dict + + @staticmethod + def parse_header_params(header_param: str) -> 'HttpHeader': + first_colon_index = header_param.index(":", 0) + key = header_param[:first_colon_index] + value = header_param[first_colon_index + 1:] + return HttpHeader(key, value) + + class RestAdapter(Adapter): safe = True supports_limit = True @@ -49,45 +76,60 @@ class RestAdapter(Adapter): @staticmethod def supports(uri: str, fast: bool = True, **kwargs: Any) -> Optional[bool]: - # return False - parsed = urllib.parse.urlparse(uri) - - if parsed.scheme not in SUPPORTED_PROTOCOLS: - return False - if fast: - return None - - # query_string = urllib.parse.parse_qs(parsed.query) - session = get_session() - response = session.head(uri) - return "application/json" in response.headers["content-type"] + return True # as this is only called from rest_api_dialect @staticmethod - def parse_uri(uri: str) -> Tuple[str, str]: + def parse_uri(uri: str) -> Tuple[str, Dict[str, List[str]], Dict[str, str], str, Dict[str, Any]]: parsed = urllib.parse.urlparse(uri) - path = urllib.parse.unquote(parsed.fragment) or "$[*]" - uri = urllib.parse.urlunparse(parsed._replace(fragment="")) - - return uri, path - - def __init__(self, uri: str, path: str = "$[*]"): + path = parsed.path + params_and_headers: Dict[str, List[str]] = urllib.parse.parse_qs(parsed.query) + fragment = urllib.parse.unquote(parsed.fragment) or "$[*]" + + headers: List[HttpHeader] = [] + query_params: Dict = {} + body: Dict = {} + for key, val in params_and_headers.items(): + if key.startswith("header"): + header: HttpHeader = HttpHeader.parse_header_params(val[0]) + headers.append(header) + if key == "body": + body = get_decoded_json_body(val[0]) + + else: + query_params[key] = val + + headers_dict = HttpHeader.load_headers(headers) + return path, query_params, headers_dict, fragment, body + + def __init__(self, + path: str, + query_params: Dict[str, List[str]], + headers_dict: Dict[str, str], + fragment: str, + body: Dict[Any, Any], + base_url: str = None, + is_https: Boolean = True, **kwargs: Any): super().__init__() - - self.uri = uri - self.path = path - + self.query_params = query_params + self.fragment = fragment + self.is_https = is_https + self.headers = headers_dict + self.body = body self._session = get_session() + if self.is_https is None or self.is_https: + prefix = "https://" + else: + prefix = "http://" + self.url = prefix + base_url + path self._set_columns() def _set_columns(self) -> None: - print("yessss boiiii") + print("custom rest adapter is being used :)") rows = list(self.get_data({}, [])) column_names = list(rows[0].keys()) if rows else [] - _, order, types = analyze(iter(rows)) - self.columns = { column_name: types[column_name]( filters=[Equal], @@ -102,7 +144,7 @@ def get_columns(self) -> Dict[str, Field]: get_cost = SimpleCostModel(AVERAGE_NUMBER_OF_ROWS) - def get_data( # pylint: disable=unused-argument + def get_data( self, bounds: Dict[str, Filter], order: List[Tuple[str, RequestedOrder]], @@ -110,12 +152,14 @@ def get_data( # pylint: disable=unused-argument offset: Optional[int] = None, **kwargs: Any, ) -> Iterator[Row]: - response = self._session.get(self.uri) + if self.body: + response = self._session.post(self.url, params=self.query_params, headers=self.headers, json=self.body) + else: + response = self._session.get(self.url, params=self.query_params, headers=self.headers) payload = response.json() - parser = JSONPath(self.path) + parser = JSONPath(self.fragment) data = parser.parse(payload) for i, row in enumerate(data): row["rowid"] = i _logger.debug(row) yield flatten(row) - diff --git a/restDbApi/rest_api_dialect.py b/restDbApi/rest_api_dialect.py new file mode 100644 index 0000000..e7e9dd2 --- /dev/null +++ b/restDbApi/rest_api_dialect.py @@ -0,0 +1,40 @@ +import urllib.parse + +from shillelagh.backends.apsw.dialects.base import APSWDialect +from sqlalchemy.engine.url import URL +from typing import Any, Dict, Tuple, List + +from sqlalchemy.pool import _ConnectionFairy + + +class RestApiDialect(APSWDialect): + name = "rest" + supports_statement_cache = True + + def create_connect_args(self, url: URL) -> Tuple[Tuple[()], Dict[str, Any]]: + str_url = url.__to_string__() + parsed = urllib.parse.urlparse(str_url) + base_url = parsed.netloc + query_params = urllib.parse.parse_qs(parsed.query) + is_https = False + if "ishttps" in query_params: + is_https = query_params["ishttps"][0] == '1' + + return (), { + "path": ":memory:", + "adapters": ["myrestadapter"], + "adapter_kwargs": { + "myrestadapter": { + "base_url": base_url, + "is_https": is_https, + }, + }, + "safe": True, + "isolation_level": self.isolation_level, + } + + def get_table_names(self, connection, schema=None, **kw) -> List[str]: + return ["'Tables' dont exists in rest APIs. Use SQL lab directly"] + + def do_ping(self, dbapi_connection: _ConnectionFairy) -> bool: + return True diff --git a/restDbApi/rest_api_engine_spec.py b/restDbApi/rest_api_engine_spec.py new file mode 100644 index 0000000..8bdedc4 --- /dev/null +++ b/restDbApi/rest_api_engine_spec.py @@ -0,0 +1,17 @@ +# Taken from: https://github.com/apache/superset/blob/master/superset/db_engine_specs/gsheets.py # noqa: E501 +from superset.db_engine_specs.sqlite import SqliteEngineSpec + + +class RestApiEngineSpec(SqliteEngineSpec): + """Engine for REST API""" + + engine = "rest" + engine_name = "REST" + allows_joins = True + allows_subqueries = True + + default_driver = "apsw" + sqlalchemy_uri_placeholder = "rest://" + + # TODO(cancan101): figure out what other spec items make sense here + # See: https://preset.io/blog/building-database-connector/ diff --git a/setup.py b/setup.py index b54e970..ef919ab 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ "myweatherapi = restDbApi.my_weather_adapter:MyWeatherAdapter", "myrestadapter = restDbApi.rest_api_adapter:RestAdapter" ], + "sqlalchemy.dialects": ["rest = restDbApi.rest_api_dialect:RestApiDialect"], }, packages=find_packages() ) diff --git a/test/rest_api_cache.sqlite b/test/rest_api_cache.sqlite index d92166fdeb51f6f1e02827dbc6c75dddefb38d6c..f70431232a8fa281461c1f79641e75afd4928004 100644 GIT binary patch delta 25530 zcmeHv33!y%xj*M4vLzvjK|&IiApud$Oul8lS%SzCP+0hXFiD^1-~>8sEKZjWZFKCfRh-DW_w+-hY- zMZhXI{km3emHFJ>3bmh{MYSxSN3|-|GFA0jS|CtnReHRE@-kB^)64uGujbLch8uI1 z`#qILnY*k)ooiX;mTp<5rCF61%4y2{hTBtKVN?YCd+)h!e$T}oe$S;02qXX(-RSkt zSt&hUuQ=$Ig~~8;Oi}vNy>pn8`Dn`ZN)}z#z$Sa$E@WN-J{;48O~h zms4S_tgovMHkFp=6lgAQRD6{CPP| zR&9NFI+b4N!&cK>>sbyZmPUh;?88#!HFeDuSVgDG#cZC}aA9;uh}0+mx`xP5T}EEc z(&omd(rfFhFr`OqrFn(eg1TlA{GSgDvMs$z%j@c{t+pCl*0fX&os3r6-JdzUnZ5fD z>N#*<%lej?Emgy|^lqxIwJNNdra%jCnbK_voBYTrrQ6No$BPU5^n6#z?461JH_(?? zvZiqp0!>!SmO--uO?eL8bX;kbIdoMw9IC%S*9&yNBuA(})v}H@$?6$Uv?QK2FtD1w?S{0NxkmXI> zlCUDsu&QNC!W{We%DD1!tG>zE-W}`qXtWwuSPl4NR``$9@y)^Ds=^7=oKweXPQS^Q zWL|WM_*&dtQ!~FV2VQ@gza?4IQnL1%My1^yCUO6d8kn3Kk2v8-(IeWta%*=lSm ztZNQ7HLNPEtH5BTWEOogfL+wHFi>7)ISVN{lR1VbiHYhP>aJTQp-GYpim!77a0wu; z(XxSV&0^Vu`$x?zV4Bs8Z8bKc*X)L7i{F)%o!+5`SeTz0kd-6a<+2synp(D`t9rg} z=Eu;32eUFmqT9rZrPi#bx`x0~YZA8C-S78^4fhdnHZ}#Cnz3LX(Q#JzOLtvWMUQ0B zqru*|KumHJFk&W$VV3)#eY|cH+QY1@4CtPKX;oB~sTCELzoN1w7qj#hvy=-A;qP{^ zf#!*Gizm&TIDziXVgoYTxTKw;TN0M8uCBj5n{7;;CAqpPM=~?M@Z;IPrG*38-iu;+ zZBff5RpYA0Q`%sbSvIliN(+0e3kI$7rs}$2i}-a?)nqHoO5$r#)ikS}Z$!h4s+krT zSP&58SS?sYG?!G(u_6p2TIN;FxA;d{nSA@;1p37wHmH9r*+n0#s?wtESuC@c_*PxD z%;I~p`VSFbYO8|bbtGac0!?zAGR!w(K-Guh03-4gnB8wq7{1wNzkxg4KS_!Qvd$H#|{7avdfBmNd` z;y2Mp^c7>oSTV;)+M%!@e`RN3GE{Dx3Uxz)-p_Jr`(ot={>Qt9^8BU!>EO>HOJo0w zB`Uf`ukU9I`e8(j+H|WflRx!?AJX>6nTvjSKO0Fey~=cc=#G+-Zru|9dtLHJvmXvz zcme*M@J$R_v^`s!aNCY-zGv2$&|7KUx-GaqrxDU&&4#+lDaaXDTs(97oQV@kr_RmE zvwxY^Tw7)};5Sn#DXU@9crp4Bf;` za!*|Wsk?Na&+v!*ouJfIm)~Qmnt(qH3}O5gH8(U=TTOvWrcLYsV92JiuBm1h5XJ@2&~!n#(^43YiP!LW1-o_v z(DZA5UEnYjhHzq_v1tX6h^EdA==Hn1!r!-rE_8bE&I&s0(Kr|6>EQe}(vRsG>PWhCyAg=~3k`Vd#qMYU_f5c$hQYhJJd;LXw2->P5rcZJ>1t8F_M>YFY%e zlKPrJup%cEhu@$e&WYW?${ir1|X2K8ADs}AHegu{n0oqvKj-9OLVv6;y`WvC`XaiP+d_S2*yRE z26Fq)0aVFE5o(9x_%B_-+7+nrF{taElyV#lhNM>o`Ot-}kH8fdEW|t1c#2PVnW|fJ z3yT;g@c8EPYpqrBtr=8g#MXwwtOt*8^2EZ~v*JmwVQOcjukP{~04|O<<$*WlQ~go0 zTUgax6L8F_#>p$L{)5Bk?j*sd2e2ddI;mzIKB-%etN$c`Zg-T~fKOT&SQg*mgRpl} z&ZmRT?KXr+Mo|{$hCF<_Ng)ZvSX~1rV}!Kg$gu8$->iDYFA)t`U0zooUk7@0^XxDf z3j6iC?mnZ-3=QLk;g18I*QNVmLZYCXUTz`AQH=vZJoN&PKForl$g$&eSkz3ZzM`-g zSH&2BkLv#D0fIBOv)MFw-e=hP7>U`qptXbeZLoJ&^kUEgq>v9{& zHbfb>@R7rD$+Yk4i&VbPLm|7eNkudOj4L!=b6$cuw zh32#Ma;Rgd#iOrT)UxFb5o<5V&u?s6>RQ=Yk02@Zw6i)`S?5|AsHqJ!G`Y&_YV%`% z%3q=7*92Bs4UPHr)%8|Qbr9iVjB6ZIg`?$Y!!4peU?aByr2BQ>#Z@(Rq2SWdPQTA( zAVZ@2J(}C&(Y(gRHFALJg5PQ)dI34~>ZVV$<2*82ioVb50q=Nh)oENDyW;4m7}jvT zFzk5YuP8*fOnG`s>I1*OXba1{_CUO(0a|6!pl4T24;=*}Xmsj9A`t6#6S0u?|cJvo4R zLkWnEB6J-7Cr>~>NOvHEafaU& zT?W6~(g}1&2J4fV34g7+&YpN|V|jCemPxj@sh0If_?{jPrcf6BGf4x~r<9)5kbZ{{nK_Bd57x7oN_T@(pr3bub zUe3zunwrw;rqae>-O8MT^(`aQd#p>|{EOM&HfM`Sxv!gulrul2RIBmprm(K5+lmD=~mu>4##~xxi{J>2eP(h1#A%$Ri=)}ux0iFCE zTcCJ+e9wI|_=$Tm=tiHCL7Vm=b5QgO<5aYt#lV$HBOYRTeAjN3wzskc{K!5B z{r*;@tam@aF6L|359W0<6Y2Bg>_Yz03zKQdU?rEf{)TOiAnM;A2BJ2sVcAss6P7`# z*C^L_H-OsL?`8%+P@2G(uXkdSKJ@M!md0!UJdM9rGyL{9*oNC$l}z9#gYq854F7Cq zZu;mYRw?mKrygP+Ir@sSDDHjxYH;uIpHHRIGGzp<_!YPToU3^Kl=c#vLCHfPUr#>H z@@V(_NYa&pp9h{NSEG^x1KiHo}P%YF4hei-W@;id-uEf^j~Zia-6}Bew1;60fMVE_8_~A{_-%(q*sf9 z0l}?;?}kyUFk<7oQlw@4WX?I=(>3ZXd9Oru1OBis|Re@AmQo z2VDHXoqZI;Em(UV-+FR5Et$zSf?k<)!!20caerPi0Bz5eX?$HMmmI%F=j>nSQ0v{` zddCCP_?o3@^ubNa)P5cBJGeEMo1VIdT}7!cv2}F#_t?Vq&oGrAdTIf^JXy)%#hZs? ziV0M_R!QWS9ZDO8!+%F?;b7ANTe4FGf8yE%I?*S81Z4HwE&6y8NHqN!mY0G%3hm${ z$w0491#5u;#{A&Z3ux_OWd1Ha0yUQO$P}r)=<`8JB318#$kp{`Q^ggE*u0;-K@oj{ z!>M+~l0 z--%oNowx-N|7eWZH%oMymZ^C7z`sg%r5ib$!S}wFNq=}2Tw3(_NIv*;iFhpASKMO3 zeUb#FA3gtJ7y$J}HkgjyjQy{FjHS}Yj7{aczR2Y}kL|hbbHR#F+FPLKcCd1u_3L4@ z;W$J?eI$ck`vV5-`~fyFf{)dCNkDX+)5wj6>`6TUU?nZIsWI3?V9qJZ?K1KmGqa4BO|2&1B?2B~-h)LzjAj;m$E_(Qh zciEeYP-?dblfai89?qYCWg0#C0ZXSLxymG({wvtrfq&gdYhPvg^w|lJ7E1RXs=GqT z<;BM{_(!cSu?W@wlJ(>Fyj;(B9s8cZh{3g|22$$@kl}`pq4?i_973>YVTQCn^u(uZ zDAoEwhtS_ahYwrxDW!oW@~J;fqZg}{B0lCG!S@^ay&E#A*IgjvORrr*`&`OaI`~g0 zy`)D#=fATFymS%J$Oj!;L}SJPqP5FzrstO_8ZUh#k1o4bxtl)tkzk3xV}@}vU7|a6 zFH_Q}H6K^UPW>I%41QLl?4twELZk0`gAJgbpR&vN_IGa~ZxB789ES5H@AZd`6ZTEm zl&6Er_vqbm;MF4>o0Ee!f6gAH58s9jdULfhjF-MQk<{%h5yH^wR~GT4zsb!|%0Uc} zZb#BTL9yhwGx^pKZ0Stp8vfkA99q*CgInM4Pxp%GwYw6} zJYhR?q2hAsABmvvrUcx~%TIzicrHQd!}q?K8q&l?wpdl;7EKYwB5%RPvj=_z8m-d!5dSlXO%|Wlr}3-Js=3teGl%LB>;hC{IvPZxH*v?*rcLy_JdddAgDa z>0HuNxsl#YQZ`fT1xh|qFXb|7oug#Z+Kb>i!U2Fi@2xDPXVVo|5=?Jct9evf(i=|J z$?J3Ioh0QJd-Q-rA*gNAvn2rjcu!@s7)8!hWhA|Jp|ZVggkLa`li&MkqI6(K zNsASW^s_c=)B7o&H0-b4?1CD)o45Y7nd*`i{kHAwalZSv4#|9!k^t5aYbUuQir%0Z z|G`3(da<(k_PzN1#tR?`qb~x?1%CnbKxG*%n+X~S5unrr=mc$%lRg`uBr*7yy_E%g z`|oo3y&r6*zU4~BZ7U(Ek7X#A(wJUs8{hfyQ0i9!;{96@m~Cn{nC<9I(66Q(DXmKy zV2J_WN6QtkKyfWFU67^Z@tvDsoZp>in}}c{tDF;MsK7xH+=20QagQ_7V*5Y9Pz(#p zPLY;yC!TVP!S6h_iH@(r(vJ;P9^#KZlS`kDhn}*pv8KX4d@)FonkrptZusNF(_*=} z00$yARcYqWKYg)a2oGsP;C!@JMeX9V@sQ@WtFbPyP#~QR6ncFMwZ6l0(-DpBfEAOo z2UkKvxadbIY$-qaY<)CfbC@eJ!scE$y9?;6RctsWiVNH}UeDnHTnKAc`UpUI3(7G9 zk8n9{b9UDuHibTfRqkq{mBRdF+a~Le7b{!36Aa@pTAL49ZGD-ES<<-kSP}o~xR|5E zjb32tlir~9$uwxSgghWz*t*g1pT-V{9OusizF$qn{O_(&is;{+z;!&lG`HL5^)4vR z1OLs|3B-Hppa*s8X)`f8OgW3yOsX52%d&C zSaN3vM}h~$Yy|)(hS_w#3rZxyTHr{Wa?mT%Bt#Uw$nhm}cR}c8I$pa0TB=pcQy7wAog1o;+%hKV=pY#K- z`Au@Bj%viiKlyVry>=tq#dl$tZu4QDg4LLIdb85NPgD+xfy&9B+>^-tyHaiFj8)3; z+vY3l)BNJl5_LD)iIQAF<_s~NP%SUqqzqi#NhzAb8b_EC_(1+Hx#q0Kp%AQl|^IWd_DdkLAtj%AlT- z!A&_s*lHdgq&O9~hc8)_HA=cwvD=kyRb+D@r0f)F5!)5PhO1{OZhF;={fZ+PzxBZk zdT@sFOL@vyJcfWddPP&Esn>&|$+MK@_8C$*$r4TO*-)!jO;&Wl#dGOomIC5j12KR_ zE2JYgfjEB~%TfU?1S}R>T#kxnG6^j@#X^ft^yn1DpvCtxAK!gnv9QSY(httT(yv~q zWXlr^VqFWky4j(4bOF9!6q>l`uZ2n$7^9R&(k5MYiP25KSL?W~34h4b(lBL;su)L<4M%9$J8*SC2)o>GBU* zy>uy-Z2>3mTL4ZzmWqSG-Z#@|TPZa0#BtDp)BXWh{`sd>axaGS1Ghnh^yD^$ADksv zr+{{hg+USmPNln~mF;YbP?i!S(AX~`G968k=tawKK=t>_ zAu3FY@!zy;b7c) zDp#nyi6V5JMN2+olLaM5(Y~+P2)F@Y&35A?JOl(^QR@Vqswt*HJ8pnlb=2c{3`>3w z!3oZlHirb{^wmPad44zrVN)L8uaUN#K3#>WHZ~}iC1c5m{pB+j=EBQfAWFA}QfXt9 zO$tZb#pyhR%K#ljTK9E|!8v8G7`d{0#h4O#*o?FYdn3=0?Pw4T+rec_0A{4m_%rI$ ztmLJ{xDW_)%8Dz9a0+TC@e!}5f(#wa9>7CW1n2f9Vw&ttpy|AA z;s^Z3qv_Olb=S_E2nyv=#dR<((#2`lArvOYzj_}rCy;h_l0HBG)sq>t?FQu`5n&oi z;*56ta^)i09)w#a5>^O29LNU&cdYVbTUVA;E0uP@RF^4-1-2=GcY+xJenACJQIqbdnn}ZT|X$ zH}XNhmJuK+&J*PMn?HId&Mv)?R)IN-a5ETO45Aq`8HxCZB*6e`xdl$-_NNdl`U$WG zdr=h?=<%y|peN}!c6gcZ-;gC4q2D%GGvp;U@yDN$qg0`v6 zX(s%iu&Xi?c{x-)6}RU!kj+W?BbH6CfIgB@5cqS#Ex-OTmUVlIf>auY!`qPI7hd#K z1`U{n(dv)j@GqJL(nko67+};KQJe62KSshrI4*MYG8A}M2`0PZ`@n_F4vOGC zJ=Y6)9dS&Mr|g48VDBeK3P0Sm{qP6ZBifnNa5MEi2va4)qJ!rmQha1Lj_H^oAX9{G zGuY=zMfdV&ZuQco|A$>E#_}acHM(;?QtgoNc9No*~Iz4oqNbn-ej1VQ0kno|u9 zZVTc#lYNl@3^-%qT5!gqw^$C&KVZ^`0f3n^=%iQq89By_xmyugOs!rh)9WAh0)gYs z=r=`1V;(`cLhcKx%qm!&6oP!Yoq{NMf;iMyD2Ikkc7+K^-7ZX`AA zWiA{JfZ4!`rES09Xkd30rlYi0)-TH=1xCvzyQJBN@yjZedd2Ielx2wVof?4{D~>HC z7l=Y*|6H)r~?8YHES$w|k1#a96je z#s2zl;J9H`%3#Igp@R2WfpG1I-o?^;h%#@7N=Ftz5ysHMK1D}G)KcMBfAIuM^!T9c zBbFW(QF_FY7O<5eBW*EellG(|-HdjA<4MCly>_4;Vpc0@y9JN_#4;s)pvMpwier?# zQ?mu6<@u#c^!irAm5FdltQI zmP}GtF#9IPoROHOO`pi%ivkHiDLeE{`&?~#kc>ed12q)_M(D6H5o2V-N%!54tkUGM zP!i&ZVk1;KQ0)suU-;1ck{9XFIAvH@_<`#h&cS31#7#NWA;-m!n?JUAtW*b*|0QnP z*Q9^j2)Q*>YE8thkRuba6yr<@`pIB7@@8tL$Bi5M@z&u*m4&v>!L5L>HxgOpo~S0WsHRqz|)M(pMla1WPRfY<1Mc21zu@Z*G%eFh4F>cEwcwSlDFrr(NBt-0}WusviWKd-U@pVbaGE zn+^Vw%%*F+VAldOp7DtyB{qz^_)!glIZWx)|K*3sXTpyZ>MJeOMqf>(k$(YwWdcsP z6Yz?pqAw(QEM7G;jh3xfrXrP+(*@hCl<5_^xr7bK_K5q~@#G1~a{pw-UdNWqC>4$* zJ)M}Wth6t3d{!jVV^b6%Q!zfVkSRMVFO#BTLHNDn4zl#&lJuG$i(b?j<3QDhUqp>& zj8azfTMy=Tjw#8wok%nWd?FYdBV2Azr^t}(6j9B>36Py4##<0VE+1(^FB3bW;(OtT z6hB!FcVg(Z{S}Xwc63+LN7!31I&A?m(Y^&p0nO;n?yzH8?Sdm>aJ$~j1niw6OP$YN zvQsSVvQq?58+NQ4X%euD#l^Lt2;VAFPLUSDTC!7Q z3-+!$kw^GZ%i+}t#W515XJuq_xWPy>o@o~oPpAEeeTSc?L3(-UdYB>^Kdm32>MGH%(9d!^50PBkl_K6+IP91jyAR@)(bh)>h1Y#aTd^D5b$fK@vtaXMP8?g%2;R|v#2LH2e=dHkZZC9cBk zV8>+ku8l}QpqoB*;iae6u@xz>L{Z5qnP3~}@q4?{8Gb6Bjfk}L^yJN$w$MG^X%eop z43&vsw*MPv8|c0y1Yln-fpw9Qs%Wqc4>H-!X`S9o;_Ut*1HG!+g?`XgzID@B`;P3+ z5(o7zQAWvkrDWa_`7+#-gV~NDhtZxoY`;t@JH-gT=p-`#fD}ZCfz-5bB19gXQi-*& zu$zVVPac^MBOz}iZJrrUO(jL00!?~n0b2um9?w;XO3{u&aU1HZ;_)ZmV~X*!8Teo*{s$0Zi7$?wgTecW}eFO!^=UTUTe##a@ZH0kt52S>vvSN#};?*slpR&bJT(&!# zy2bNTwivSF)uh_AJ$>Ec`6*ir%@r{hQKJ@lf#fV7(>O0>i>Q7MKXG)fb&KbxY%zrG zhf7n>v2OAFlr4t7aou99gm_Q|jIzbK`&hSb=Tf;Ss?TLr<7rA4Wg(t6T9ogJC;h~G z3B!$Xf%^KI>T*t+3q{J;!ov+QJY2Pgp{QC5 z|KM)>L6oZDm)KIZa3^zgxLj>g1-e8^)P@@q@PE`>b|}LPwbx<(@04LCi1#M!cN^!# zeYX*<;S$W={Rj7WJH;-+T(vv41haSM;2v*%@xNPwDW8Be)bmz?X?(*H%#!Gr0`cX$ z63p;3od2Q{%nQ0-*{%DP_pQ6Ud(p4*`{A*^PnLY!H*8UYSUT{#$NIi|tna(W`s_y| zzk93?4^hOt{_x#neI32{?;h)menR!T$NKE&g1&pK4=*vE@iDiq9>DwVu|Dwx@^_E* kb$Ffm|L4d0VjhDE$>**A^^f$mz0r5&w|}G$@$PZ|4~Bk1nE(I) delta 109 zcmV-z0FwWJzz~3-3XmHG4gdfEDUl#Q0S>WXqz?`Q5Ags1000*c`w#XH@v(tJ1G7~V zjSLTMVQFq@ZfBHEj0=~6e}W* diff --git a/test/testDialect.py b/test/testDialect.py new file mode 100644 index 0000000..ccf1a1b --- /dev/null +++ b/test/testDialect.py @@ -0,0 +1,43 @@ +import urllib.parse + +from sqlalchemy import create_engine + +engine = create_engine("rest://api.weatherapi.com?ishttps=1") +connection = engine.connect() +for i in connection.execute('SELECT * FROM "/v1/forecast.json?key=f1f6d01fe66c4cd48de113426232402&q=London&days=5#$.forecast.forecastday[*]"'): + print(i) + + +engine = create_engine("rest://stg.wsp-store-info.walmart.com?ishttps=0") +connection = engine.connect() + +endpoint = '/wsp-store-info/v1/layers/pipelinestores?pipelinestores=APPROVED_LX,IDEA_LX,PROJECTS_LX&hlat=34.51010643175928&hlong=-98.35332961466173&llat=30.660521215372583&llong=-117.87603469278673'; +headers = '&header1=Content-Type:application/json' +headers += '&header2=WM_CONSUMER.ID:7aa13f7b-5e29-451b-bcfa-ba1b62471b89' +headers += '&header3=WM_SVC.ENV:stg' +headers += '&header4=WM_SVC.NAME:WSP-STORE-INFO' +headers += '&header5=sessionId:czBuMDJxbQ==' +jsonpath = "#$[*]" + +virtual_table = endpoint+headers+jsonpath +print(virtual_table) +for i in connection.execute(f'SELECT * FROM "{virtual_table}"'): + print(i) + +engine = create_engine("rest://stg.wsp-store-info.walmart.com?ishttps=0") +connection = engine.connect() + +endpoint = '/wsp-store-info/v1/layers/remodel?hlat=60.22131924306147&hlong=-50.04269584580887&llat=1.006947387589554&llong=-160.11325248643387&openstores=SUP'; +headers = '&header1=Content-Type:application/json' +headers += '&header2=WM_CONSUMER.ID:7aa13f7b-5e29-451b-bcfa-ba1b62471b89' +headers += '&header3=WM_SVC.ENV:stg' +headers += '&header4=WM_SVC.NAME:WSP-STORE-INFO' +headers += '&header5=sessionId:czBuMDJxbQ==' +body = '{ "remodelInitiative": ["EOTF", "RSDWH", "D8 PET COOLER"], "remodelProjectStatus": ["RECENTLY_COMPLETED"] }' +jsonpath = "#$.remodelData[*]" + +encoded_body = "&body="+urllib.parse.quote(body, 'utf8') +virtual_table = endpoint+headers+encoded_body+jsonpath +print(virtual_table) +for i in connection.execute(f'SELECT * FROM "{virtual_table}"'): + print(i) \ No newline at end of file