From 5276216c48158f935ae79fe312521e957ff7ac89 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sat, 21 May 2016 22:07:37 -0700 Subject: [PATCH 01/14] Carapal react mockup This is really just a mock up written in React to try different components. It could become scaffolding to build a prototype, or not. --- caravel/assets/javascripts/carapal.jsx | 160 +++++++++++++++++++++++++ caravel/assets/stylesheets/carapal.css | 58 +++++++++ caravel/templates/caravel/carapal.html | 6 + 3 files changed, 224 insertions(+) create mode 100644 caravel/assets/javascripts/carapal.jsx create mode 100644 caravel/assets/stylesheets/carapal.css create mode 100644 caravel/templates/caravel/carapal.html diff --git a/caravel/assets/javascripts/carapal.jsx b/caravel/assets/javascripts/carapal.jsx new file mode 100644 index 0000000000000..9213dc1926e11 --- /dev/null +++ b/caravel/assets/javascripts/carapal.jsx @@ -0,0 +1,160 @@ +import React from 'react'; +import { render } from 'react-dom'; + +import { Button, ButtonGroup } from 'react-bootstrap'; +import { Table } from 'reactable'; + +import brace from 'brace'; +import AceEditor from 'react-ace'; +import 'brace/mode/sql'; +import 'brace/theme/chrome'; + + +var ReactGridLayout = require('react-grid-layout'); + +require('../stylesheets/carapal.css') + +const GridItem = React.createClass({ + getDefaultProps: function() { + return { + nopadding: false + }; + }, + render: function () { + return ( +
+
{this.props.header}
+
+ {this.props.children} +
+
+ ) + } +}); + +const SqlEditor = React.createClass({ + render: function () { + return ( + Query + + + + + }> +
+ +
+ + ) + } +}); + +const ResultSet = React.createClass({ + render: function () { + return ( + + Result Set + + + + + + + + )} + nopadding={true}> + + + ) + } +}); + + +const Workspace = React.createClass({ + render: function () { + return ( + + + Tables + + + + Queries + + + + ) + } +}); + +const App = React.createClass({ + render: function () { + return ( + +
+ +
+
+ +
+
+ +
+
+ +
+ + ) + } +}); + +render( + , + document.getElementById('app') +); diff --git a/caravel/assets/stylesheets/carapal.css b/caravel/assets/stylesheets/carapal.css new file mode 100644 index 0000000000000..fb55bf43e2540 --- /dev/null +++ b/caravel/assets/stylesheets/carapal.css @@ -0,0 +1,58 @@ + +.nopadding { + padding: 0px; +} +.panel { + height: 100%; + width: 100%; + overflow: auto; +} +.panel-body { +} + +.react-grid-layout { + position: relative; + transition: height 200ms ease; +} +.react-grid-item { + transition: all 200ms ease; + transition-property: left, top; +} +.react-grid-item.cssTransforms { + transition-property: transform; +} +.react-grid-item.resizing { + z-index: 1; +} + +.react-grid-item.react-draggable-dragging { + transition: none; + z-index: 3; +} + +.react-grid-item.react-grid-placeholder { + background: red; + opacity: 0.2; + transition-duration: 100ms; + z-index: 2; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + -o-user-select: none; + user-select: none; +} + +.react-grid-item > .react-resizable-handle { + position: absolute; + width: 20px; + height: 20px; + bottom: 0; + right: 0; + background: url('data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBzdGFuZGFsb25lPSJubyI/Pg08IS0tIEdlbmVyYXRvcjogQWRvYmUgRmlyZXdvcmtzIENTNiwgRXhwb3J0IFNWRyBFeHRlbnNpb24gYnkgQWFyb24gQmVhbGwgKGh0dHA6Ly9maXJld29ya3MuYWJlYWxsLmNvbSkgLiBWZXJzaW9uOiAwLjYuMSAgLS0+DTwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+DTxzdmcgaWQ9IlVudGl0bGVkLVBhZ2UlMjAxIiB2aWV3Qm94PSIwIDAgNiA2IiBzdHlsZT0iYmFja2dyb3VuZC1jb2xvcjojZmZmZmZmMDAiIHZlcnNpb249IjEuMSINCXhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHhtbDpzcGFjZT0icHJlc2VydmUiDQl4PSIwcHgiIHk9IjBweCIgd2lkdGg9IjZweCIgaGVpZ2h0PSI2cHgiDT4NCTxnIG9wYWNpdHk9IjAuMzAyIj4NCQk8cGF0aCBkPSJNIDYgNiBMIDAgNiBMIDAgNC4yIEwgNCA0LjIgTCA0LjIgNC4yIEwgNC4yIDAgTCA2IDAgTCA2IDYgTCA2IDYgWiIgZmlsbD0iIzAwMDAwMCIvPg0JPC9nPg08L3N2Zz4='); + background-position: bottom right; + padding: 0 3px 3px 0; + background-repeat: no-repeat; + background-origin: content-box; + box-sizing: border-box; + cursor: se-resize; +} diff --git a/caravel/templates/caravel/carapal.html b/caravel/templates/caravel/carapal.html new file mode 100644 index 0000000000000..fb8f7cba429f3 --- /dev/null +++ b/caravel/templates/caravel/carapal.html @@ -0,0 +1,6 @@ +{% extends "caravel/basic.html" %} + +{% block tail_js %} + {{ super() }} + +{% endblock %} From 3e45c4e11d14c01687d81f30450802746068bce9 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Mon, 13 Jun 2016 16:12:28 -0700 Subject: [PATCH 02/14] Preliminary commit for Celery backend --- .gitignore | 2 ++ caravel/bin/caravel | 6 +++++ caravel/config.py | 18 +++++++++++++++ .../versions/33459b145c15_allow_temp_table.py | 23 +++++++++++++++++++ caravel/models.py | 1 + setup.py | 1 + 6 files changed, 51 insertions(+) create mode 100644 caravel/migrations/versions/33459b145c15_allow_temp_table.py diff --git a/.gitignore b/.gitignore index 4c93a874bad0e..eb93aa3c20e7e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ dist caravel.egg-info/ app.db *.bak +.idea +*.sqllite # Node.js, webpack artifacts *.entry.js diff --git a/caravel/bin/caravel b/caravel/bin/caravel index 3ae2b68516da3..fb823fa83de97 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -127,5 +127,11 @@ def refresh_druid(): session.commit() +@manager.command +def worker(): + """Starts a Caravel worker for async query load""" + raise NotImplementedError("# TODO! @b.kyryliuk") + + if __name__ == "__main__": manager.run() diff --git a/caravel/config.py b/caravel/config.py index 79c87d0a9215a..b96d7edb060b6 100644 --- a/caravel/config.py +++ b/caravel/config.py @@ -180,6 +180,23 @@ # Set this API key to enable Mapbox visualizations MAPBOX_API_KEY = "" +# Default celery config is to use SQLA as a broker, in a production setting +# you'll want to use a proper broker as specified here: +# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html +""" +# Example: +class CeleryConfig(object): + BROKER_URL = 'amqp://guest:guest@localhost:5672//' + ## Broker settings. + BROKER_URL = 'amqp://guest:guest@localhost:5672//' + CELERY_IMPORTS = ('myapp.tasks', ) + CELERY_RESULT_BACKEND = 'db+sqlite:///results.db' + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} +""" +CELERY_CONFIG = None + +# Maximum number of rows returned in the SQL editor +SQL_MAX_ROW = 1000 try: from caravel_config import * # noqa @@ -188,3 +205,4 @@ if not CACHE_DEFAULT_TIMEOUT: CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT') + diff --git a/caravel/migrations/versions/33459b145c15_allow_temp_table.py b/caravel/migrations/versions/33459b145c15_allow_temp_table.py new file mode 100644 index 0000000000000..7fab1a13a63e3 --- /dev/null +++ b/caravel/migrations/versions/33459b145c15_allow_temp_table.py @@ -0,0 +1,23 @@ +"""allow_temp_table + +Revision ID: 33459b145c15 +Revises: d8bc074f7aad +Create Date: 2016-06-13 15:54:08.117103 + +""" + +# revision identifiers, used by Alembic. +revision = '33459b145c15' +down_revision = 'd8bc074f7aad' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column( + 'dbs', sa.Column('allow_temp_table', sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column('dbs', 'allow_temp_table') diff --git a/caravel/models.py b/caravel/models.py index 5c46293d6b809..3325f18af6905 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -378,6 +378,7 @@ class Database(Model, AuditMixinNullable): sqlalchemy_uri = Column(String(1024)) password = Column(EncryptedType(String(1024), config.get('SECRET_KEY'))) cache_timeout = Column(Integer) + allow_temp_table = Column(Boolean, default=False) extra = Column(Text, default=textwrap.dedent("""\ { "metadata_params": {}, diff --git a/setup.py b/setup.py index d02d1499f1634..ceb266d9fef18 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ zip_safe=False, scripts=['caravel/bin/caravel'], install_requires=[ + 'celery==3.1.23', 'cryptography==1.4', 'flask-appbuilder==1.8.1', 'flask-cache==0.13.1', From b0674c8ac9df00cca452645c117b2272d44bc8b2 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Sat, 25 Jun 2016 18:09:27 -0700 Subject: [PATCH 03/14] Move the SQL query execution to the celery worker. --- .idea/vcs.xml | 6 ++++++ caravel/bin/caravel | 14 +++++++++++++- caravel/config.py | 13 +++++++++---- caravel/models.py | 13 +++++++++++++ caravel/tasks.py | 44 ++++++++++++++++++++++++++++++++++++++++++ celery_results.sqlite | Bin 0 -> 60416 bytes celerydb.sqlite | Bin 0 -> 19456 bytes 7 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 .idea/vcs.xml create mode 100644 caravel/tasks.py create mode 100644 celery_results.sqlite create mode 100644 celerydb.sqlite diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000000000..94a25f7f4cb41 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/caravel/bin/caravel b/caravel/bin/caravel index fb823fa83de97..ccc3f0d85c233 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -5,6 +5,8 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import celery +from celery.bin import worker as celery_worker from datetime import datetime from subprocess import Popen import textwrap @@ -130,7 +132,17 @@ def refresh_druid(): @manager.command def worker(): """Starts a Caravel worker for async query load""" - raise NotImplementedError("# TODO! @b.kyryliuk") + # celery -A tasks worker --loglevel=info + print("Broker url: ") + print(config.get('CELERY_CONFIG').BROKER_URL) + application = celery.current_app._get_current_object() + c_worker = celery_worker.worker(app=application) + options = { + 'broker': config.get('CELERY_CONFIG').BROKER_URL, + 'loglevel': 'INFO', + 'traceback': True, + } + c_worker.run(**options) if __name__ == "__main__": diff --git a/caravel/config.py b/caravel/config.py index b96d7edb060b6..d86f61c17947e 100644 --- a/caravel/config.py +++ b/caravel/config.py @@ -179,6 +179,8 @@ # Set this API key to enable Mapbox visualizations MAPBOX_API_KEY = "" +# Maximum number of rows returned in the SQL editor +SQL_MAX_ROW = 1000 # Default celery config is to use SQLA as a broker, in a production setting # you'll want to use a proper broker as specified here: @@ -193,10 +195,13 @@ class CeleryConfig(object): CELERY_RESULT_BACKEND = 'db+sqlite:///results.db' CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} """ -CELERY_CONFIG = None - -# Maximum number of rows returned in the SQL editor -SQL_MAX_ROW = 1000 +class CeleryConfig(object): + ## Broker settings. + BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' + CELERY_IMPORTS = ('caravel.tasks', ) + CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite' + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} +CELERY_CONFIG = CeleryConfig try: from caravel_config import * # noqa diff --git a/caravel/models.py b/caravel/models.py index 3325f18af6905..02fb5da177c1b 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -1702,3 +1702,16 @@ class FavStar(Model): class_name = Column(String(50)) obj_id = Column(Integer) dttm = Column(DateTime, default=func.now()) + + +class Query(Model): + """Used to store the information related to the executed queries.""" + # TODO(@b.kyryliuk): add indexes + __tablename__ = 'query' + id = Column(Integer, primary_key=True) + sql = Column(Text) + user_id = Column(Integer, ForeignKey('ab_user.id')) + start_dttm = Column(DateTime, default=datetime.now, nullable=True) + finish_dttm = Column(DateTime, default=None, nullable=True) + tmp_table_id = Column(Integer) # TODO(@b.kyryliuk): add a foregin key to link the query with the tmp table + diff --git a/caravel/tasks.py b/caravel/tasks.py new file mode 100644 index 0000000000000..38f55b2511f9d --- /dev/null +++ b/caravel/tasks.py @@ -0,0 +1,44 @@ +import celery +from caravel import db, models, app +from sqlalchemy import select, text +from sqlalchemy.sql.expression import TextAsFrom +import pandas as pd + +print str(app.config.get('CELERY_CONFIG').BROKER_URL) +celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG')) + +# TODO @b.kyryliuk - move it into separate module as it should be used by the celery module +@celery_app.task +def get_sql_results(database_id, sql): + """Gets sql results from a Caravel database connection""" + # TODO @b.kyryliuk handle async + # handle models.Queries (userid, sql, timestamps, status) index on userid, state, start_ddtm + session = db.session() + mydb = session.query(models.Database).filter_by(id=database_id).first() + + content = "" + if mydb: + eng = mydb.get_sqla_engine() + if app.config.get('SQL_MAX_ROW'): + sql = sql.strip().strip(';') + qry = ( + select('*') + .select_from(TextAsFrom(text(sql), ['*']).alias('inner_qry')) + .limit(app.config.get('SQL_MAX_ROW')) + ) + sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) + try: + df = pd.read_sql_query(sql=sql, con=eng) + content = df.to_html( + index=False, + na_rep='', + classes=( + "dataframe table table-striped table-bordered " + "table-condensed sql_results").split(' ')) + except Exception as e: + content = ( + '
' + "{}
" + ).format(e.message) + session.commit() + return content diff --git a/celery_results.sqlite b/celery_results.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..d202c5adb61b71b097777073489a4e266b4adfd7 GIT binary patch literal 60416 zcmeHwOOxDImR>#dBBgFW#%V!NtKn@9&)Z=%){oBJNDmY!MV4=is%2!^4Ar*y$V`9DEo5{|Nt=e~$3s zwek!6dszG2_VK$1-M{#^-#Px@2d^J~ad7k>kN^1SpC5j4bieKLSN{9I%3pi)-48xE z{Pph?!FV2r@jT8h{36It7jY5neEIq(5AWZ5e82Pf-j5&L@9g}rbMyMOBN9i)|{2o^I_|wns zE0P(OJr2Ut&g1((S2ua``KLd9a=&xa-0ar%TOYsv#=AfL;P4<>M)9-!=^Q)C57tGh zzV>$>%(r&F@y1V%-yoiSsaR%yJ&&KR<7F7{eEHfI%XWTva~Uk+5AmC~e)Ufe-+b4y z4*w9W-+m2wTi`l~M0^!3%BD-K+7;5UK;*Xb#*-X<{`rygK`@cL2<7Z*KDw1@WU%yurSu$Q1@%=1Iv+GX}pPan+ zNxGaQ)Au^*_^UW9IC!$K+|CincUK7hgK3=K)_bSxoLDEe(_LNQij#OcHQ&D-tX8)Ry6MU4;>L%a zd4f!W?e;$GB+HKazRH8(rk&2i&c|3!aTd(+VX|J{d_Tyh`G=kN-#f zEzS2=Sz3VQapV(od|xdQ%Ufs5P9D!EC)uSde3HjmetVowqhRTuUSt>ZWPN%&UByd3 zPuE!(`&qn7!M-31&f@vzU&^24bza$OJyYZhkl4U4QFbO*i%BLOaQ7rsTjZp+5$R(gQKc(~DJS6}Bouls^9Q~`Ke|P-P zAoK3g-J|L8zd8OF$8R6~Ieu{U=Qoc7H;utI)h$e1;U+V^AHz10G zZeaDIZa;D+PH#Bshfe=GUD%7=zB?TBx&zm;p;n<$$8kLB_U&jqjs}kFM8P$>&*?uY1nmbYtkLr5tQ)29`r|1Z{&vFTXf;17xeA16Vsw_Vc^)^U^p0coxyNCnT*`g z#E##j3*%7`Il-vc9eNX(2(LHnj)Fnd^_W_p2SN8wP{%?CI z{~y`^>`rGtKk|A_{{Nd~`@e$MJUBM~e=N|jZT`PyojAkMz#Bb(aJ2FNC7Spf_5V$v z;`OXy|H}WrJT&<>`u|t*-=Lo!yk_KI;)=D&zuiCS_xc03|NO!6M*bxV<(rm&QdA?$ z9oSd$e zKYpVW|3s1{{$|Br>nX42^?O(1|Lqa~A8Z%?{?g?CTWbH`G73OaCu&dtgF9C5jy*W( z^@l^R|NPOLr2?2rG|D%t0F0*ctl^d8{{l?HtDE@yOOyZZ!D08{zj71$%75p+3L$`fOjQ_dy z_4xlkJUIN}!JobI(q4Oj2iJ|@OHe0mf)6a6HR|^U*7KiTD+NE5Xp(PM@QsFo>AwQ~ zeX9kZ+yAdc{*Mm+_~7VE{1w*(=#JsWljx({#Z3Q}qH#Bh&k`U>`44~-o+Znve;Qvj{Itm801dB8 zYC+v4xeBr(Av|*R1ANNVs&RL-?2n3Z8eN!g>W{dJx@zw}J=}sH!v%5gc)&>`U*dVc z{@v^9E#0|yAW^~9Tl(zkE!AIqNN*6(OEUz4(I!`^TRG5_n>m(?AkyY>KgGFz55 zUvxKEMp>F*4N(99`&sO-EdR`|uEHlnVV9IUtgdRK<@je#byYhpuUB1l2`#s}s+E@I z37*lFJ)`GUH{4!UyaG zzqnY%0g0@SNe+GpjSy77-g=XEj>h^PhWc`i<1rMWgkSw?D|0NXwkvmT zkZM%_({+-8co?Q@(Mv^#NOj{KlEpf%e%87ur(fIN+Z3krB!Yohpj88VBY=4m(6b@` zzttaAS8mhP)K*zMi3{3P)9S+3CEC4Kf%s~*&S(BIEs{wR2FCQ1#<5k03|imAJe~UM zWwL~AUXxs52Q7oMWGW>+Y?ZNPd|ryyWS*Y;5@qL%YFBOZab^4KY>Ign7ycs7bG#w} zBQ2{{d9ECPHA{=s&u2jl}?L>v^2}!8D6wp`LQxR!y44(VAor zgFqnRkUOZ45bK4MBcRoqdbYzQ${#(-hVLe(Qj zbBQH#J+4O7vgW~h3ERBqlAHnGG+CCQSI&>nv6;-3)m+uKLhFWcd$nz4cK5R+U4X&_ z0Rc2-Qe0Fos+|rug=4bX5N;?AY!H&a^~#6hh3(N#UAuETU5{~poTjHF6NskPq}lMy zVUpuz?kCoL_DaU<91V#%90ApZ=WD+jjDXnq0md{H0!G+m|Q{ip$e5(L$b11PgYCwfCTR0LVcz$MniziXs2sgDnZ5 z{A*XXFOLg0dM4q#kv5ALH1H|&lcnAji*#Ac=1|Dw2TLT$ace9xt1YSL7t7Emv%Eki z6dNP=&yx6D!h#XAP|82{)5R2bN zhkO$i>FGr(AP?wmbyb}_9P0u-)*?x@+Z4enA6L}zsU?@uYuRVY{fcOSBGPQ+(1^B` z)1o%2NIpp=gscXG0WAi7enE^$&zDHsoWazu5vp1;Mp+FSr;TJ3PXhF*BMuABpynwx zAz-G(vnC8*k{q7~?npV2ZQ{+d;KZP^| z7%ZvM5cmfJ5v3_79)=?CUqG~cRX8|@IrJ%~j-8zQvtltvWH-MMtQh(7>MEn&Qsbz? zpDGD|?Hl#MD;2>=t{Fce%cf zV&nj@xlyzVi_6lgkpI6$g{gW0!0EH?h{zKYQjil^LDi8IA^o%jAJ&U8rE&L4Z4a0t ziI7GO3q%T#7Z?EltY&|(kDBeV#r@VO$yI86f|A~h6go3v>A?R7{aauX)@|kpJJNQV*1lsUI{dA-VQ+F57yN zn@Bc_>(s8QKaP5o9QCC+5|T)A!nIeWfLw3$MT*>w>_s@dR$JSP?YG#2Q_f|2>P2^` zF`!keuoOe>2~_39HA&(*9OpF(PLg%my#7_Afc;5;7r{HRhfEVq2)m>z zuhTWvHO(tAs1tZ|V?q$@2TuSZv098)MMzg+iY4J$0I#F+k7!$39NIBitE?S@IZB94 zvtTuoDKFP%_HL@cNQn-xFnY9rV~is|j*Nt-lOKhf;i7s$6AZ!h{mW>ofwKoK$F$%c4O24)m?U4%M;eQ~&h058Nxb3k$ zQ4zO$nj@4LMb$ORU-~v^YHoi9+eV5QYJ2kx)us5^+s>xC-gs@Iyu zH0v0O(uD4kx7xpK)h=`zVAaKi8s&>rV@NZXRlXNhcu8f#wwbus9>Uw957}B_pfiwD zGU_8%L#h$)8%!t42LSydTMK;xrN0oc-o9)<+5-qv@C#siwwR;HLjHe8gVWpIs?8jb z+LE`W$Unz`;~M)~cZ$X9c{QRt;v=T&^6XVb)#;Jt-w_!x>0yI%MNSct-Rrr~uu9Oi zibe82_w7N?kAjOF22tw(!8^2g#6io~H}SS_0)!qmT(*55nY#6w)vdlw-~UCNLHDBG z0;wb!HOfn{erHsB^+qSg^A*njLedEs8(kSuP8R`sBouijZdRk1#d9zLl|oZ<5m0n_ z%rzs}tikZ&Unp$-9x<}J1U5+s^X#rRT*@iJ&I6o8A@c;Z9PUzTTI5yk=UKTtd{Cr9 zQi(KU9PDLaJYfsuAII zJ5g=GZX>5l$!1-!aLXb?MN05aEMB713#8mF_F&z~Qm%x6rrRJe#LK;LQ!cNds>Mlt z#|D!}>!>-n>XGu32?AK9)|7ClLbCW_ZEpJLCM&|Lqap}=Bh_bRioxYIJN3Yu{8F+! zOVF=CE^5AzykDCGfww8nq0Esd24YeKaBz5rYMUOpLR5;?k1yCQ4u^y9Qp}OOnL^bP zDqQC4Q2zh#ZSwy{Y13=jaI(Uqi1L@ts98Vcvp8F!4TBy#CXX1sZwa-rh^V?ef2*Fo zPBo=Eq@gNdC(B$Zisu;}UWsTJN{n`oP7z3UkTYbC5^(i&W(J%uylO!I^Qo?#NtVZJ zxvvVR7m#9uLO{|PFl`2sFz`A!lPX1vCsROX zQ~%rdK`onQeCyG`C0d+p<^&i3vQiN|2<$`=n#5|@&jx_*ZxxrR z-A3>NTyEi$1^9mkTa1%eROKzBA}2gp2|1^_NNYioEu>F^i?u;^RTtn>a{%n|h#Cl{ z6=552s0&!7Qab`d+*WSWS=E-?x*AL^t8P_36vaRWPgS{W3jU5ayON~OTZt;y{5*!1 zQs~mA-A1c0tkVl}c>7SzbSs~46;sb1p&22v7yju|v_CAj#v=3ZDe?hT2W!QfRt{R+ z*##)GMx=I+ONekSxkaQFHMJ74c|upMWS(G$3ZBH_4wL%*73BZ#!-J(@Qb1-0Y3(q7@F3MtHJu5UCYOuvnYpo-ZRRMBVUg zFcJNFT28r5A=0wWP`APTz;6?tNOy6n7EaMy#;G?Id#l%sTDI!kgN7}wdUY8-HS0v4 zLneNKR@o3q!6JG(uw!CiJU2bkxqvpWm?+vVViba?WhuOC9R!G!k;kieh90bF%|F<$ zL6VoV6j5@rOTbd;Y)hUQ2hLTTsa}rh)NWv8b#>D_g8bi5J^l`Fqo8$~$_8ZmES4ym zW<9kE*7g5E5c&DN-7 zAxpp>qAFknj_#_i(mQmEsM)GnuvDFXV+Bl>bo#iwqWfaPlGYBFW121_Oyx9LNmLmT%=M@s=*~k2Q&rjGT)WA>h4m?<0<7RlwPT} zG2`2YXEjZx_c|KyrVvVWFCgR%$teOXSvn!Y;RT|0dVXEmwy%))Ezx__4x7pDOLT?Z zXC=>hSxxQrcdDsc8>4_IU?Ii@RaccfguPW|T4%SEO3>bvAQcFcgB-OteeV6RpqCCft#}36c zz%!Jw#()K&%Vny|=7#%4RAnhqzUR|Dc0kY!+6F z>r5gFr(zmos7s`}ZGtF5^$VqLoqr?iS*@g%GzWEpGRy{WPkK|(6q6X#0na0<6gb>TGSmQRT(4-yQGa4)K)tg0H6UPtGv4r`@l1dbQjqDj#= z`Ryv58l_ti5#;}eEyC~eaXW^rp58_hrIJd59cs1iP(5hJH16pkZAjuHz=A#uCL~le zK+2c-csTa5T8vSGT7fj&hVnxg`vPl*;d#}s;Zw6R(o)n{k>gr)yMuAD>q?JB0TVq%RiW8AL5^%xcIbgF?MGERhBVUmc zr4fGuigttL(l2eR{kv|p+AWsDODAS|rMg!Ye0p9r#1!saEv#vM_cl+7mSo|)mKKoQ zS2PaOXj#E?^I1YQk_LB4c}#SvVJxkHpvQ5~HHrrMt;a^rgLA!YAph@O?*H@oHZ`KE zVtk?PHu$hJTYBrV=sTqC(g|FRS^WpT8lp+v4Ec{xyKaBHA1G7aBUtP3i|1}F|h~LGKYK~+om#EY{`ek4;pGn7%S*=?c-4swtC6c^a zKRz`(A^;axS8Y8dAs%QXZTT|FCX!22z-214bxUvi`~GSUtnf2q-zCn6&4&OOP8#iL z{K*M2xgxX%4mfUMfFTIi?imdkl~k;(R=>{7+PlnsZh1paLpnmcG{9dVK7@OZyfAcw z$oXhx-$uJnkdhMmZ|-)H6Y_tHMttkeSXE2fE7rCe+`$XPazNgY<3i&AN$pUQzItV? zJKN*=r>*89kt*d-hfQv!j?yhrR)S?=oPBkb7C0ig0UVj6iW*xri?-scmJMR@fl`Vh z!B$bZ1~9FK8Y??ZR#8C6;`987Pzuvb z874o0N|WdtmxJLzXC0~{_kN1O?lVsjZ%|EIBnM;?Q~%EgSmS(-{Ge#hx!f%VNhq7u zoC-t0ALK^1;b~J_xOyD|_Bp&T2*PF7X0z)=9OQ`*kTQju2|DM1P(;abkBZHxdH|zf zK~E38))NvS#wD5yb~cpYWE@^o!h9EKD?-Ep)=7nwa*^aKs@Z+jI`F9_Jm8@u=<166 z7SvkTZ1LLt6vzFbvzm^v$P!UX-{LMf-J~43wpRoEux3mQcd4hc>RK;7nsS%nXKJt= zEQo*~4tFZC)O-dI1XAs!dQrWiR)9UqHZD)&lr-x=!K{!tgAVTi?OXNl$&?FA z--9$^X=q$S9G;HTXMz>l95_g1YAN~sw5Zcy!aa=Rz=>PbH%Xo%-!`g@w2#@FrcwjXMm50pA3JccbRQA~IcEc+tVzp)h0$|C`adLTj5X16K zBAHTp=3R@nw`13`_eOV6tF&tTpaiHYfSScgWYOXi2C+H>wBhN6DWpoc&qH=25xJRU z0@GXDgk(z=IAKdb+T)PTQrPH)=!0;KD!+g`Nz!A6U^P*P{C9UeKbucHS9N(ce+GjU zScP&Q#c4NqTXQ4_XkIH!kjdZ3YCSd|eGMn1K=~NhJHUyk6t8l`bU+OBydE=(PP`^Y z>@dM?@SLa(p^M@~w@AAPw7W#RD~#1Gj)PfYx;6F{^hSB+1yz6rYvXu1s>E(4E<-SN z2WV~n(3Z0zaiJVu@WM@K#t-#a`D2ukPPa6GVBKy*(#RJwqh3(pqMT z9V^khc0<6W$*55jdS+TZwJ2bk%VBA&8cn4I_atd?P)A7;Ejko|yxdB8V>ig+Fg=WW zQ1J&m3G#Lk7^icX=OoWX$4Oc)Pi%lL&f~)nZm5#UBzflJ(B2jGw9(Lwb&HTe{(Huk(t6V3z}zOvV)p{Fwd7UnVr^b8znrR!1-xY2MAbO>_5BI?w_4u1 z-H<|Rcuj_=>eH%IZlUUM*dLav)nQqZH}3JK9~V)lfHLo11mPJD4ODg!c{!x~Dd~%1 zidMPOh7GsD!v*CE$OfUJ;__aT=xmNtq~%^h)G$fdlxEjAj&Xqv{c?0Za!`>1L<(~l zPDAPmBvA|W&(b+gVBQ#HrO{0@gvg@Vyyt+pL2)PG}y#iH>Pi@?x4D1MoA5dQ7%$6i&T|Vdm6b+!6 z=IcXZ~p%BXS}wK`X|; z#ziORLpZ_2=B<<9<~GC!H9>VKtQFurp=%r4Td-uCL$6B8Xpp1GL_oyl^GQLZ+>_=I zF=2wZ(KA5P2c?5)4=P5MDnYWghGYQNMq_%(wM68#r2~YG@(yo+z!Ot_23;$$9+U!s zR|77@F1OuK86lzzO5cW9w4Rp%`FG4AUOE^dmAlW_dYgCotHrDPDBBPgNW;^PrF6H< z`}P_#`w24Qp;?fNI7k zi<=%s4YRDar1&09SuE^-s39G*CCG5#gbx@{N`KS&C&rSB92Tdciup|Ym*WKGg-t1I zlf~QG0it_Yyq5{@Liw;uctmBVq{)H^c4)%}h5WmQCh9w9@)9Q>H$`g$3IO2n4O9+$ zF+tuRAbCkb#%-uMJu?YMN?~Y|N9S`1E6u!b0c+F=%w2jCx6LqNW6ywzku=%Z99C0by2?qb zbT1*K_el(KC$dg=qjC2o>+k`(po^mmApc&j2JupqK@8TzAcF-wq%0C0KHzY`q9(39 zA_9r%gDQ~S6UYPa=&li?@sed*28JACy#k6BpV+}Uh`_9Ay$qEBp?FLRH0Yw)`#b`zT&N`wEZ_E2~h)$38(~GH<#4L zAki0Y)ANE5&Dp2;K~1YLovLcg%6_RMn|Ivgzz1%V*LF%n871Dd&2tqEL?QnJl|M8O zF~Od%~2SeDC+T8f;lnxktp#n~&1>u5#MF?=+#=HLJ)fT=b^g9R+*MSnwB$#C`f z;-$Jxyv#ANgWMcC&YRo`)!HXmo!z7hRnZzxiIgRkud!dHKnxBU%XJBSLk;pj#3+M&oYspL+k|1-{IAp1jJfQa8r0QrUV0A#k}i$8d(hssN}*z<8U_ z&2J(Omczo+86oAV8D#^Ij71L5ps3rJl#|w+*pVBC36^g0Xi?ELV-Zyl1Ay6*CjMtS zTO*)q*Zl%x@u}F0hr!I-YEDIM^-govk@d>6q1pd{>$ ztrHnhi|6V0x`l8u7O&;&yFmU&zIhnAUg4cehvj7=O80@*hsFz*Y4LgvT4>=e=*KrcsW^9TH-va=l*YBk zpvE)Qeb0^ZgZ$rtx@_C@4sVCbsq#LdIEs#9na7y>D|lQErlLX4gqkVaFE>lNWH~Me z{Sh{#BtaWfbR3G!D=P)2WmKm@A@(nD)_|F(BdL8itPJfXS_R4#tIY$=0%Xb#Rg`FH z5(cn_K(kXZRls?h1FrOliPDJcHSUqNcevx!L@{3`tCeh^kIg$^&9G&-L-3%H`Il%w zms>fFyumCqa8bq1hLz;!3YdqCLv@vQD+px)_e33bN%?ZX$?|g=wzkfyOYy1Mia0z! zM50tib6SJKyC|g=)i}T>_nHRyU#HHBP6%EmN?YTCglGBXGrK#F)A-B zk!FWu?SONk*c=UqXks&6jskwT9ICZLi8P^v5NmV-9hxLaKreiR@!D(Jr^CnVQSm{| z5jD9aRjnr37L9v!d9~|2R@>WDxK;P2RH|JrgXgmg<&cv>H=i#og=pKrrvSPY+Khb=vLqyY`@C4enUp9m_iLyuRbP-+}xe9{l?Q`tKF{^N0W8 zd+%I3c=G+{|N6J@9ZcLf>f3?Ub*#Yaxa`nKc(}PbHW|ppu-hELM$GXa}kTILj~>l;~V6^bdevOF7I?Q%18f5U9D>} zv%9J%d{!OKIJ--!8L~U4>O3a%BfM*6Z}4w*e0n8OEK{0&%1AS=vmU#mhp_#p)hVC??KVp?~RS0U|@_4gqbCtG^c7`6()J z;$$9ocgcL1wbmVIpIVf~a0K3HM(2U!Rlz4o=l*Ge%mhjan5;%L5d^Fx5vk4dWWgK> YQ5tOnAMre%qb_O^KTSM^y#KZT3pP@a>i_@% literal 0 HcmV?d00001 diff --git a/celerydb.sqlite b/celerydb.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..56b518a3d96dc1e8af5c5195b719eaa9b830a77a GIT binary patch literal 19456 zcmeHO$#3IWdY4R>m#XSaCX=8rhcpEDp>sv=(Y< zTW(8g%%Ov%4q5(-9COJn*C5Ct%S;YQfC(_kAvq)nW-yomvH(ede3V>{?Q(T0Y0T7O zJxEW|dwknR{oeO|zwhy|UhPi8Of)(fc#{lGt*6rI)VDI3R4Vm8yuSwT=V=|jB=6@b z{rt-pe7>I&fBXmU3ICqjNd0C?_!Z#~*S?pYrq|)e3u(Hs`6&B1{rN9-hdi8ir$_zK zz&|@0gr}$8adRB@> zx|Bqw0J3b94LvCxd$V}t@k~Q+L%$b&FM0@1PP?OF=FrGit1$Sr%1_S1v+(GDW|cI^ zAL#jyGrP}6E}NNM^kD`+d9tXinTHK4oAk3&{P@OAU-Y(l0Smp{4xXn*ZH!&}n(=6-Qxl(cFQgfOs9_O-jOPx+YI%9vpBahfp zo5cNUK38a9yJqK)Kl%7DchKx>b~B&Lw+?e7r)`eBmNaaWNgUeR*ey(^Imeym;%3ny zolk>-HO(Q(&bQ2tUz&EEmU$9Tz1)bKWw%|KIfG))gZbo}lAWU#j6IpVZEKqEOLle; z_k;Sxt`zL9F*m5$Xr5Oaxt`f@CT5|2?#vK#OJ%9ruw$c8mfV&zGv>Lj(bf%+!3_GD zd96eSxSB0CbzUmYJ*wAi3-_FMrNamIX?3_9vpV20FKY9kHH{Lv@zxZnI%^Sqmz>`Ui1ZIL{caDlq!;hL=;G}$QTMlmHV0~g@|ZjL@4rc(()u6 z$Frl!C~1R~2&-82MIJ`JNIht+N+A<9rV_V3qwX+D8h!>d?T${m zljJW5RL9QA=xhRWJnDywKjl*OCZmPZpyLgX7wf+$O+<6q;b6p*I&!sa;w=`7Jnlz4 zVj_-wA~JyFFFb(ktBFPb^V^VpC2{b7ZWFTiHXwVrfON@Dt&O!aoWBApD*1H^N^De_L*YPK(_dxR z3MT8fR)bh5K^kSkt^YLrC{Gj4 zAV(|EPi5@3&1peD&DwPf0atMXT!4K5TtqM^61R1Jwj9$qbBT@Gz(e>&m~-32{(%Ma z1m2=;js2E=3LL-~=JmAEmrKB>#LzC`0_@yiLV-&d0#{%~ZT*lRvX2sH_qF=}JP2i# zU?c{S?~5#=TvS8gk0PHU9LchV$qW9!M%6G#NCKpVRHVKfi9QQd$fzv)h)J08WMP-? z`p?q;_X)-TawPg#4n&3&E_#$GqOTy?qgp@$jjZthg?rkwl<-pgU)oo1DE=GA=SIu! zIrBc?ztVLN5i)wbTW!Q0z&zj|@_Bjc^s>Yll|Gm-pmsq3_MFYSG2o}U;;3YN2VM8OZ)`&&EroJlvzmrPA>#a}E(E1`a z@c0&ee(6qqX*_oleSR4m$d8V#xbn1dnCrW3ckB>50-ZYdTKo)j-VpFs}?^)dVg-F=`}?Zf)aEi_rJ zVRveWxbHS9ooXW~3nos_VFr|$jY@3Ht*+bBr*5w^Efkvvz~Q4{5XYdG&BODYojX|Y z{L@0*1SDAVz_et2IWh zorO6%ZQTMJrQ}f#wgjIC7Q|jG6IYKvdj0yJa7oo9PXjKWcz}Nnz>!rfa*2eL`Jsm+ z^{v7`^+?DlQ$(&Pp#KBHMPK!)sL7h51yn(VN_P(b_g;ekD{RXgZdPfH^B5X)y0Qo;U*oj{@}q5=5!Ar5ZhKT=c$aUzMT z4~_$6OolT91SkYQk~Q$L5mOd>#%sYpoTX5ah9fZwVTR13il{}v1V}-oa0N@mqwLP% zfAb~yzY?o?C-6ToN_yhg&%y7XyMs!v+ORstygYFl$Fv4z=DgE&2aaSGzz!JLEBr^u z0~a5(K8<`Zmw_r1JyiizgF=rYjcE~CoaMQC{wIvUHBe<}qk^AJkrIl&M|}~F{U{hI zG88QKjMswyKnp^O2ossAP?1p#+Z60@Vi@3vf`vjo&%bl{-?|0=Nf`M`DED2!e|_o@ zhdMp8XXY{3_c7qVJ~LVl*#DJ&at6?xmnrxFeWTDv&U}Uc9SmyGrvVegPy_rEP@Ed! za0n1cG+EeQIm=PtzM?!5hvLMDTz zM54kl9G7Jf)$?I15M{!BpDUD6fD6R(uIhgb{xwk(V$Ju!2*3>J0>(hkGEZTVD&q*f zM*rXUSjZvjA2JpAf0+aIN5HiS1D~t1hbW2e9{%sY6#th8!QKV@TOGILAg5s^`oH7! zEa3mGjxz`TzoDOl|4-b4IWXo~h!MCwWA3i-@4-=dkAeR8dXACl*VPG{Pjrytt4vieWs{V)Pe-@7by~Pp0XY9cO``^44 z`%7&9D`!N%H1_-dx7eQ<@m~3-5&x5%fgu?s;Q#YzYLNOA^gpS!nh^hM&YT4&0I`44 zBya`*;({yvzoP+#2Tv~{5zg#jP?s9`^1zWLbO4-~lLJEd3+I2{X#EdHz7&xFi+H)A zaU^P>_!CdShj0eWEKaKdm zH84wd-|SU75bK*8EgdiQKj6Pr1e*YzLmUv*8pkuR4SI0xO8-Y-EX%~hV8%lj5A?qz zdLEbqNK=@m5;)h#Uf}=UX#9s##Ib}VF;XH;WEzFwgbJ<=*0eAR;IT!_`Twr}{~Ohe B;TQk_ literal 0 HcmV?d00001 From fad15aa342dd5302fa6197c34356258988fc5696 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Wed, 29 Jun 2016 14:24:56 -0700 Subject: [PATCH 04/14] React scetch --- caravel/assets/=0.20.0 | 0 caravel/assets/javascripts/carapal.jsx | 150 ++++-- caravel/assets/javascripts/table.jsx | 87 +++ .../assets/stylesheets/fixed-data-table.css | 509 ++++++++++++++++++ caravel/templates/caravel/table.html | 6 + caravel/views.py | 6 + 6 files changed, 722 insertions(+), 36 deletions(-) create mode 100644 caravel/assets/=0.20.0 create mode 100644 caravel/assets/javascripts/table.jsx create mode 100644 caravel/assets/stylesheets/fixed-data-table.css create mode 100644 caravel/templates/caravel/table.html diff --git a/caravel/assets/=0.20.0 b/caravel/assets/=0.20.0 new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/caravel/assets/javascripts/carapal.jsx b/caravel/assets/javascripts/carapal.jsx index 9213dc1926e11..0944ee87254da 100644 --- a/caravel/assets/javascripts/carapal.jsx +++ b/caravel/assets/javascripts/carapal.jsx @@ -1,7 +1,7 @@ import React from 'react'; import { render } from 'react-dom'; -import { Button, ButtonGroup } from 'react-bootstrap'; +import { Button, ButtonGroup, Tab, Tabs } from 'react-bootstrap'; import { Table } from 'reactable'; import brace from 'brace'; @@ -9,11 +9,56 @@ import AceEditor from 'react-ace'; import 'brace/mode/sql'; import 'brace/theme/chrome'; - var ReactGridLayout = require('react-grid-layout'); - require('../stylesheets/carapal.css') +// Data Model +var query_result_data=[ + {'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, + {'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, + {'State': 'Colorado', + 'Description': 'new description that shouldn\'t match filter', + 'Tag': 'old'}, + {'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, + {'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, + {'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, + {'State': 'Colorado', + 'Description': 'new description that shouldn\'t match filter', + 'Tag': 'old'}, + {'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, + {'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, + {'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, + {'State': 'Colorado', + 'Description': 'new description that shouldn\'t match filter', + 'Tag': 'old'}, + {'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, + ] + +var table_preview_data=[ + {'id': '1', 'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, + {'id': '2', 'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, + {'id': '3', 'State': 'Colorado', + 'Description': 'new description that shouldn\'t match filter', + 'Tag': 'old'}, + {'id': '4', 'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, + {'id': '5', 'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, + {'id': '6', 'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, + {'id': '7', 'State': 'Colorado', + 'Description': 'new description that shouldn\'t match filter', + 'Tag': 'old'}, + ] + +var table_schema_data=[ + {'column': 'id', 'type': 'Integer'}, + {'column': 'State', 'type': 'String'}, + {'column': 'Description', 'type': 'String'}, + ] + +var saved_queries_data=[] +var query_history_data=[] + + +// Componenst const GridItem = React.createClass({ getDefaultProps: function() { return { @@ -32,18 +77,38 @@ const GridItem = React.createClass({ } }); -const SqlEditor = React.createClass({ +const TabbedSqlEditor = React.createClass({ render: function () { return ( - Query + Tabbed Query "Bla Bla Bla" }> +
+ + + + + + + + + + + +
+ + ) + } +}); + +const SqlEditor = React.createClass({ + render: function () { + return (
- ) } }); -const ResultSet = React.createClass({ +const QueryResultSet = React.createClass({ render: function () { return ( + data={this.props.data} + /> ) } }); +const TabbedResultsSet = React.createClass({ + render: function () { + return ( + +
+ + + + + + + + + + + +
+ + +
+ + +
+ + ) + } +}); const Workspace = React.createClass({ render: function () { @@ -130,6 +203,7 @@ const Workspace = React.createClass({ } }); + const App = React.createClass({ render: function () { return ( @@ -137,14 +211,18 @@ const App = React.createClass({ className="layout" cols={12} rowHeight={30} width={window.innerWidth}> -
- -
- +
-
- +
+
diff --git a/caravel/assets/javascripts/table.jsx b/caravel/assets/javascripts/table.jsx new file mode 100644 index 0000000000000..e513257a89caa --- /dev/null +++ b/caravel/assets/javascripts/table.jsx @@ -0,0 +1,87 @@ +import React from 'react'; +import { render } from 'react-dom'; + +import { Button, ButtonGroup, Tab, Tabs } from 'react-bootstrap'; + +import brace from 'brace'; +import AceEditor from 'react-ace'; +import 'brace/mode/sql'; +import 'brace/theme/chrome'; + + +var ReactGridLayout = require('react-grid-layout'); +var FixedDataTable = require('fixed-data-table'); + +require('../stylesheets/carapal.css') +require('../stylesheets/fixed-data-table.css') + +const MyTextCell = React.createClass({ + render: function() { + const {rowIndex, field, data, ...props} = this.props; + return ( + + {data[rowIndex][field]} + + ); + } +}); + +const QueryResultSetV2 = React.createClass({ + getInitialState: function() { + return { + myTableData: [ + {name: 'Rylan', email: 'Angelita_Weimann42@gmail.com'}, + {name: 'Amelia', email: 'Dexter.Trantow57@hotmail.com'}, + {name: 'Estevan', email: 'Aimee7@hotmail.com'}, + {name: 'Florence', email: 'Jarrod.Bernier13@yahoo.com'}, + {name: 'Tressa', email: 'Yadira1@hotmail.com'}, + ], + }; + }, + render: function () { + return ( + + Name} + cell={ + + } + width={100} + height={30} + /> + Email} + cell={ + + } + width={100} + height={30} + /> + + ); + } +}); + +const App = React.createClass({ + render: function () { + return ( + + ) + } +}); + +render( + , + document.getElementById('app') +); diff --git a/caravel/assets/stylesheets/fixed-data-table.css b/caravel/assets/stylesheets/fixed-data-table.css new file mode 100644 index 0000000000000..60f40a78a99ff --- /dev/null +++ b/caravel/assets/stylesheets/fixed-data-table.css @@ -0,0 +1,509 @@ +/** + * FixedDataTable v0.6.1 + * + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableCellGroupLayout + */ + +.fixedDataTableCellGroupLayout_cellGroup { + -webkit-backface-visibility: hidden; + backface-visibility: hidden; + left: 0; + overflow: hidden; + position: absolute; + top: 0; + white-space: nowrap; +} + +.fixedDataTableCellGroupLayout_cellGroup > .public_fixedDataTableCell_main { + display: inline-block; + vertical-align: top; + white-space: normal; +} + +.fixedDataTableCellGroupLayout_cellGroupWrapper { + position: absolute; + top: 0; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableCellLayout + */ + +.fixedDataTableCellLayout_main { + border-right-style: solid; + border-right-width: 1px; + border-width: 0 1px 0 0; + box-sizing: border-box; + display: block; + overflow: hidden; + position: absolute; + white-space: normal; +} + +.fixedDataTableCellLayout_lastChild { + border-width: 0 1px 1px 0; +} + +.fixedDataTableCellLayout_alignRight { + text-align: right; +} + +.fixedDataTableCellLayout_alignCenter { + text-align: center; +} + +.fixedDataTableCellLayout_wrap1 { + display: table; +} + +.fixedDataTableCellLayout_wrap2 { + display: table-row; +} + +.fixedDataTableCellLayout_wrap3 { + display: table-cell; + vertical-align: middle; +} + +.fixedDataTableCellLayout_columnResizerContainer { + position: absolute; + right: 0px; + width: 6px; + z-index: 1; +} + +.fixedDataTableCellLayout_columnResizerContainer:hover { + cursor: ew-resize; +} + +.fixedDataTableCellLayout_columnResizerContainer:hover .fixedDataTableCellLayout_columnResizerKnob { + visibility: visible; +} + +.fixedDataTableCellLayout_columnResizerKnob { + position: absolute; + right: 0px; + visibility: hidden; + width: 4px; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableColumnResizerLineLayout + */ + +.fixedDataTableColumnResizerLineLayout_mouseArea { + cursor: ew-resize; + position: absolute; + right: -5px; + width: 12px; +} + +.fixedDataTableColumnResizerLineLayout_main { + border-right-style: solid; + border-right-width: 1px; + box-sizing: border-box; + position: absolute; + z-index: 10; +} + +body[dir="rtl"] .fixedDataTableColumnResizerLineLayout_main { + /* the resizer line is in the wrong position in RTL with no easy fix. + * Disabling is more useful than displaying it. + * #167 (github) should look into this and come up with a permanent fix. + */ + display: none !important; +} + +.fixedDataTableColumnResizerLineLayout_hiddenElem { + display: none !important; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableLayout + */ + +.fixedDataTableLayout_main { + border-style: solid; + border-width: 1px; + box-sizing: border-box; + overflow: hidden; + position: relative; +} + +.fixedDataTableLayout_header, +.fixedDataTableLayout_hasBottomBorder { + border-bottom-style: solid; + border-bottom-width: 1px; +} + +.fixedDataTableLayout_footer .public_fixedDataTableCell_main { + border-top-style: solid; + border-top-width: 1px; +} + +.fixedDataTableLayout_topShadow, +.fixedDataTableLayout_bottomShadow { + height: 4px; + left: 0; + position: absolute; + right: 0; + z-index: 1; +} + +.fixedDataTableLayout_bottomShadow { + margin-top: -4px; +} + +.fixedDataTableLayout_rowsContainer { + overflow: hidden; + position: relative; +} + +.fixedDataTableLayout_horizontalScrollbar { + bottom: 0; + position: absolute; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableRowLayout + */ + +.fixedDataTableRowLayout_main { + box-sizing: border-box; + overflow: hidden; + position: absolute; + top: 0; +} + +.fixedDataTableRowLayout_body { + left: 0; + position: absolute; + top: 0; +} + +.fixedDataTableRowLayout_fixedColumnsDivider { + -webkit-backface-visibility: hidden; + backface-visibility: hidden; + border-left-style: solid; + border-left-width: 1px; + left: 0; + position: absolute; + top: 0; + width: 0; +} + +.fixedDataTableRowLayout_columnsShadow { + width: 4px; +} + +.fixedDataTableRowLayout_rowWrapper { + position: absolute; + top: 0; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule ScrollbarLayout + */ + +.ScrollbarLayout_main { + box-sizing: border-box; + outline: none; + overflow: hidden; + position: absolute; + -webkit-transition-duration: 250ms; + transition-duration: 250ms; + -webkit-transition-timing-function: ease; + transition-timing-function: ease; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.ScrollbarLayout_mainVertical { + bottom: 0; + right: 0; + top: 0; + -webkit-transition-property: background-color width; + transition-property: background-color width; + width: 15px; +} + +.ScrollbarLayout_mainVertical.public_Scrollbar_mainActive, +.ScrollbarLayout_mainVertical:hover { + width: 17px; +} + +.ScrollbarLayout_mainHorizontal { + bottom: 0; + height: 15px; + left: 0; + -webkit-transition-property: background-color height; + transition-property: background-color height; +} + +/* Touching the scroll-track directly makes the scroll-track bolder */ +.ScrollbarLayout_mainHorizontal.public_Scrollbar_mainActive, +.ScrollbarLayout_mainHorizontal:hover { + height: 17px; +} + +.ScrollbarLayout_face { + left: 0; + overflow: hidden; + position: absolute; + z-index: 1; +} + +/** + * This selector renders the "nub" of the scrollface. The nub must + * be rendered as pseudo-element so that it won't receive any UI events then + * we can get the correct `event.offsetX` and `event.offsetY` from the + * scrollface element while dragging it. + */ +.ScrollbarLayout_face:after { + border-radius: 6px; + content: ''; + display: block; + position: absolute; + -webkit-transition: background-color 250ms ease; + transition: background-color 250ms ease; +} + +.ScrollbarLayout_faceHorizontal { + bottom: 0; + left: 0; + top: 0; +} + +.ScrollbarLayout_faceHorizontal:after { + bottom: 4px; + left: 0; + top: 4px; + width: 100%; +} + +.ScrollbarLayout_faceVertical { + left: 0; + right: 0; + top: 0; +} + +.ScrollbarLayout_faceVertical:after { + height: 100%; + left: 4px; + right: 4px; + top: 0; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTable + * + */ + +/** + * Table. + */ +.public_fixedDataTable_main { + border-color: #d3d3d3; +} + +.public_fixedDataTable_header, +.public_fixedDataTable_hasBottomBorder { + border-color: #d3d3d3; +} + +.public_fixedDataTable_header .public_fixedDataTableCell_main { + font-weight: bold; +} + +.public_fixedDataTable_header, +.public_fixedDataTable_header .public_fixedDataTableCell_main { + background-color: #f6f7f8; + background-image: -webkit-linear-gradient(#fff, #efefef); + background-image: linear-gradient(#fff, #efefef); +} + +.public_fixedDataTable_footer .public_fixedDataTableCell_main { + background-color: #f6f7f8; + border-color: #d3d3d3; +} + +.public_fixedDataTable_topShadow { + background: 0 0 url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAECAYAAABP2FU6AAAAF0lEQVR4AWPUkNeSBhHCjJoK2twgFisAFagCCp3pJlAAAAAASUVORK5CYII=) repeat-x; +} + +.public_fixedDataTable_bottomShadow { + background: 0 0 url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAECAYAAABP2FU6AAAAHElEQVQI12MwNjZmZdAT1+Nm0JDWEGZQk1GTBgAWkwIeAEp52AAAAABJRU5ErkJggg==) repeat-x; +} + +.public_fixedDataTable_horizontalScrollbar .public_Scrollbar_mainHorizontal { + background-color: #fff; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableCell + */ + +/** + * Table cell. + */ +.public_fixedDataTableCell_main { + background-color: #fff; + border-color: #d3d3d3; +} + +.public_fixedDataTableCell_highlighted { + background-color: #f4f4f4; +} + +.public_fixedDataTableCell_cellContent { + padding: 8px; +} + +.public_fixedDataTableCell_columnResizerKnob { + background-color: #0284ff; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableColumnResizerLine + * + */ + +/** + * Column resizer line. + */ +.public_fixedDataTableColumnResizerLine_main { + border-color: #0284ff; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule fixedDataTableRow + */ + +/** + * Table row. + */ +.public_fixedDataTableRow_main { + background-color: #fff; +} + +.public_fixedDataTableRow_highlighted, +.public_fixedDataTableRow_highlighted .public_fixedDataTableCell_main { + background-color: #f6f7f8; +} + +.public_fixedDataTableRow_fixedColumnsDivider { + border-color: #d3d3d3; +} + +.public_fixedDataTableRow_columnsShadow { + background: 0 0 url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAABCAYAAAD5PA/NAAAAFklEQVQIHWPSkNeSBmJhTQVtbiDNCgASagIIuJX8OgAAAABJRU5ErkJggg==) repeat-y; +} +/** + * Copyright (c) 2015, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @providesModule Scrollbar + * + */ + +/** + * Scrollbars. + */ + +/* Touching the scroll-track directly makes the scroll-track bolder */ +.public_Scrollbar_main.public_Scrollbar_mainActive, +.public_Scrollbar_main:hover { + background-color: rgba(255, 255, 255, 0.8); +} + +.public_Scrollbar_mainOpaque, +.public_Scrollbar_mainOpaque.public_Scrollbar_mainActive, +.public_Scrollbar_mainOpaque:hover { + background-color: #fff; +} + +.public_Scrollbar_face:after { + background-color: #c2c2c2; +} + +.public_Scrollbar_main:hover .public_Scrollbar_face:after, +.public_Scrollbar_mainActive .public_Scrollbar_face:after, +.public_Scrollbar_faceActive:after { + background-color: #7d7d7d; +} diff --git a/caravel/templates/caravel/table.html b/caravel/templates/caravel/table.html new file mode 100644 index 0000000000000..b8326db4c604e --- /dev/null +++ b/caravel/templates/caravel/table.html @@ -0,0 +1,6 @@ +{% extends "caravel/basic.html" %} + +{% block tail_js %} + {{ super() }} + +{% endblock %} diff --git a/caravel/views.py b/caravel/views.py index da306510e64b3..c1c3efe634f1f 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -1419,6 +1419,12 @@ def sqlanvil(self): """SQL Editor""" return self.render_template('caravel/sqllab.html') + @has_access + @expose("/table") + def table(self): + """SQL Editor""" + return self.render_template('caravel/table.html') + appbuilder.add_view_no_menu(Caravel) if config['DRUID_IS_ACTIVE']: From d549e9be127462a0aa722f551c193cfb6937a884 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Tue, 12 Jul 2016 17:50:00 -0700 Subject: [PATCH 05/14] Refactor SQL execution to use the celery if configured. --- .idea/vcs.xml | 6 - caravel/assets/=0.20.0 | 0 caravel/assets/javascripts/carapal.jsx | 238 -------- caravel/assets/javascripts/table.jsx | 87 --- caravel/assets/stylesheets/carapal.css | 58 -- .../assets/stylesheets/fixed-data-table.css | 509 ------------------ caravel/bin/caravel | 5 +- caravel/config.py | 11 +- .../versions/33459b145c15_allow_temp_table.py | 23 - caravel/models.py | 14 - caravel/tasks.py | 13 +- caravel/templates/caravel/carapal.html | 6 - caravel/templates/caravel/table.html | 6 - caravel/views.py | 6 - celery_results.sqlite | Bin 60416 -> 0 bytes celerydb.sqlite | Bin 19456 -> 0 bytes 16 files changed, 11 insertions(+), 971 deletions(-) delete mode 100644 .idea/vcs.xml delete mode 100644 caravel/assets/=0.20.0 delete mode 100644 caravel/assets/javascripts/carapal.jsx delete mode 100644 caravel/assets/javascripts/table.jsx delete mode 100644 caravel/assets/stylesheets/carapal.css delete mode 100644 caravel/assets/stylesheets/fixed-data-table.css delete mode 100644 caravel/migrations/versions/33459b145c15_allow_temp_table.py delete mode 100644 caravel/templates/caravel/carapal.html delete mode 100644 caravel/templates/caravel/table.html delete mode 100644 celery_results.sqlite delete mode 100644 celerydb.sqlite diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f4cb41..0000000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/caravel/assets/=0.20.0 b/caravel/assets/=0.20.0 deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/caravel/assets/javascripts/carapal.jsx b/caravel/assets/javascripts/carapal.jsx deleted file mode 100644 index 0944ee87254da..0000000000000 --- a/caravel/assets/javascripts/carapal.jsx +++ /dev/null @@ -1,238 +0,0 @@ -import React from 'react'; -import { render } from 'react-dom'; - -import { Button, ButtonGroup, Tab, Tabs } from 'react-bootstrap'; -import { Table } from 'reactable'; - -import brace from 'brace'; -import AceEditor from 'react-ace'; -import 'brace/mode/sql'; -import 'brace/theme/chrome'; - -var ReactGridLayout = require('react-grid-layout'); -require('../stylesheets/carapal.css') - -// Data Model -var query_result_data=[ - {'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, - {'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, - {'State': 'Colorado', - 'Description': 'new description that shouldn\'t match filter', - 'Tag': 'old'}, - {'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, - {'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, - {'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, - {'State': 'Colorado', - 'Description': 'new description that shouldn\'t match filter', - 'Tag': 'old'}, - {'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, - {'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, - {'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, - {'State': 'Colorado', - 'Description': 'new description that shouldn\'t match filter', - 'Tag': 'old'}, - {'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, - ] - -var table_preview_data=[ - {'id': '1', 'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, - {'id': '2', 'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, - {'id': '3', 'State': 'Colorado', - 'Description': 'new description that shouldn\'t match filter', - 'Tag': 'old'}, - {'id': '4', 'State': 'Alaska', 'Description': 'bacon', 'Tag': 'renewed'}, - {'id': '5', 'State': 'New York', 'Description': 'this is some text', 'Tag': 'new'}, - {'id': '6', 'State': 'New Mexico', 'Description': 'lorem ipsum', 'Tag': 'old'}, - {'id': '7', 'State': 'Colorado', - 'Description': 'new description that shouldn\'t match filter', - 'Tag': 'old'}, - ] - -var table_schema_data=[ - {'column': 'id', 'type': 'Integer'}, - {'column': 'State', 'type': 'String'}, - {'column': 'Description', 'type': 'String'}, - ] - -var saved_queries_data=[] -var query_history_data=[] - - -// Componenst -const GridItem = React.createClass({ - getDefaultProps: function() { - return { - nopadding: false - }; - }, - render: function () { - return ( -
-
{this.props.header}
-
- {this.props.children} -
-
- ) - } -}); - -const TabbedSqlEditor = React.createClass({ - render: function () { - return ( - Tabbed Query "Bla Bla Bla" - - - -
- }> -
- - - - - - - - - - - -
- - ) - } -}); - -const SqlEditor = React.createClass({ - render: function () { - return ( -
- -
- ) - } -}); - -const QueryResultSet = React.createClass({ - render: function () { - return ( - - Result Set - - - - - - -
- )} - nopadding={true}> -
- - ) - } -}); - -const TabbedResultsSet = React.createClass({ - render: function () { - return ( - -
- - - - - - - - - - - -
- - -
- - -
- - ) - } -}); - -const Workspace = React.createClass({ - render: function () { - return ( - - - Tables - - - - Queries - - - - ) - } -}); - - -const App = React.createClass({ - render: function () { - return ( - -
- -
-
- -
-
- -
- - ) - } -}); - -render( - , - document.getElementById('app') -); diff --git a/caravel/assets/javascripts/table.jsx b/caravel/assets/javascripts/table.jsx deleted file mode 100644 index e513257a89caa..0000000000000 --- a/caravel/assets/javascripts/table.jsx +++ /dev/null @@ -1,87 +0,0 @@ -import React from 'react'; -import { render } from 'react-dom'; - -import { Button, ButtonGroup, Tab, Tabs } from 'react-bootstrap'; - -import brace from 'brace'; -import AceEditor from 'react-ace'; -import 'brace/mode/sql'; -import 'brace/theme/chrome'; - - -var ReactGridLayout = require('react-grid-layout'); -var FixedDataTable = require('fixed-data-table'); - -require('../stylesheets/carapal.css') -require('../stylesheets/fixed-data-table.css') - -const MyTextCell = React.createClass({ - render: function() { - const {rowIndex, field, data, ...props} = this.props; - return ( - - {data[rowIndex][field]} - - ); - } -}); - -const QueryResultSetV2 = React.createClass({ - getInitialState: function() { - return { - myTableData: [ - {name: 'Rylan', email: 'Angelita_Weimann42@gmail.com'}, - {name: 'Amelia', email: 'Dexter.Trantow57@hotmail.com'}, - {name: 'Estevan', email: 'Aimee7@hotmail.com'}, - {name: 'Florence', email: 'Jarrod.Bernier13@yahoo.com'}, - {name: 'Tressa', email: 'Yadira1@hotmail.com'}, - ], - }; - }, - render: function () { - return ( - - Name} - cell={ - - } - width={100} - height={30} - /> - Email} - cell={ - - } - width={100} - height={30} - /> - - ); - } -}); - -const App = React.createClass({ - render: function () { - return ( - - ) - } -}); - -render( - , - document.getElementById('app') -); diff --git a/caravel/assets/stylesheets/carapal.css b/caravel/assets/stylesheets/carapal.css deleted file mode 100644 index fb55bf43e2540..0000000000000 --- a/caravel/assets/stylesheets/carapal.css +++ /dev/null @@ -1,58 +0,0 @@ - -.nopadding { - padding: 0px; -} -.panel { - height: 100%; - width: 100%; - overflow: auto; -} -.panel-body { -} - -.react-grid-layout { - position: relative; - transition: height 200ms ease; -} -.react-grid-item { - transition: all 200ms ease; - transition-property: left, top; -} -.react-grid-item.cssTransforms { - transition-property: transform; -} -.react-grid-item.resizing { - z-index: 1; -} - -.react-grid-item.react-draggable-dragging { - transition: none; - z-index: 3; -} - -.react-grid-item.react-grid-placeholder { - background: red; - opacity: 0.2; - transition-duration: 100ms; - z-index: 2; - -webkit-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - -o-user-select: none; - user-select: none; -} - -.react-grid-item > .react-resizable-handle { - position: absolute; - width: 20px; - height: 20px; - bottom: 0; - right: 0; - background: url('data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBzdGFuZGFsb25lPSJubyI/Pg08IS0tIEdlbmVyYXRvcjogQWRvYmUgRmlyZXdvcmtzIENTNiwgRXhwb3J0IFNWRyBFeHRlbnNpb24gYnkgQWFyb24gQmVhbGwgKGh0dHA6Ly9maXJld29ya3MuYWJlYWxsLmNvbSkgLiBWZXJzaW9uOiAwLjYuMSAgLS0+DTwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+DTxzdmcgaWQ9IlVudGl0bGVkLVBhZ2UlMjAxIiB2aWV3Qm94PSIwIDAgNiA2IiBzdHlsZT0iYmFja2dyb3VuZC1jb2xvcjojZmZmZmZmMDAiIHZlcnNpb249IjEuMSINCXhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHhtbDpzcGFjZT0icHJlc2VydmUiDQl4PSIwcHgiIHk9IjBweCIgd2lkdGg9IjZweCIgaGVpZ2h0PSI2cHgiDT4NCTxnIG9wYWNpdHk9IjAuMzAyIj4NCQk8cGF0aCBkPSJNIDYgNiBMIDAgNiBMIDAgNC4yIEwgNCA0LjIgTCA0LjIgNC4yIEwgNC4yIDAgTCA2IDAgTCA2IDYgTCA2IDYgWiIgZmlsbD0iIzAwMDAwMCIvPg0JPC9nPg08L3N2Zz4='); - background-position: bottom right; - padding: 0 3px 3px 0; - background-repeat: no-repeat; - background-origin: content-box; - box-sizing: border-box; - cursor: se-resize; -} diff --git a/caravel/assets/stylesheets/fixed-data-table.css b/caravel/assets/stylesheets/fixed-data-table.css deleted file mode 100644 index 60f40a78a99ff..0000000000000 --- a/caravel/assets/stylesheets/fixed-data-table.css +++ /dev/null @@ -1,509 +0,0 @@ -/** - * FixedDataTable v0.6.1 - * - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableCellGroupLayout - */ - -.fixedDataTableCellGroupLayout_cellGroup { - -webkit-backface-visibility: hidden; - backface-visibility: hidden; - left: 0; - overflow: hidden; - position: absolute; - top: 0; - white-space: nowrap; -} - -.fixedDataTableCellGroupLayout_cellGroup > .public_fixedDataTableCell_main { - display: inline-block; - vertical-align: top; - white-space: normal; -} - -.fixedDataTableCellGroupLayout_cellGroupWrapper { - position: absolute; - top: 0; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableCellLayout - */ - -.fixedDataTableCellLayout_main { - border-right-style: solid; - border-right-width: 1px; - border-width: 0 1px 0 0; - box-sizing: border-box; - display: block; - overflow: hidden; - position: absolute; - white-space: normal; -} - -.fixedDataTableCellLayout_lastChild { - border-width: 0 1px 1px 0; -} - -.fixedDataTableCellLayout_alignRight { - text-align: right; -} - -.fixedDataTableCellLayout_alignCenter { - text-align: center; -} - -.fixedDataTableCellLayout_wrap1 { - display: table; -} - -.fixedDataTableCellLayout_wrap2 { - display: table-row; -} - -.fixedDataTableCellLayout_wrap3 { - display: table-cell; - vertical-align: middle; -} - -.fixedDataTableCellLayout_columnResizerContainer { - position: absolute; - right: 0px; - width: 6px; - z-index: 1; -} - -.fixedDataTableCellLayout_columnResizerContainer:hover { - cursor: ew-resize; -} - -.fixedDataTableCellLayout_columnResizerContainer:hover .fixedDataTableCellLayout_columnResizerKnob { - visibility: visible; -} - -.fixedDataTableCellLayout_columnResizerKnob { - position: absolute; - right: 0px; - visibility: hidden; - width: 4px; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableColumnResizerLineLayout - */ - -.fixedDataTableColumnResizerLineLayout_mouseArea { - cursor: ew-resize; - position: absolute; - right: -5px; - width: 12px; -} - -.fixedDataTableColumnResizerLineLayout_main { - border-right-style: solid; - border-right-width: 1px; - box-sizing: border-box; - position: absolute; - z-index: 10; -} - -body[dir="rtl"] .fixedDataTableColumnResizerLineLayout_main { - /* the resizer line is in the wrong position in RTL with no easy fix. - * Disabling is more useful than displaying it. - * #167 (github) should look into this and come up with a permanent fix. - */ - display: none !important; -} - -.fixedDataTableColumnResizerLineLayout_hiddenElem { - display: none !important; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableLayout - */ - -.fixedDataTableLayout_main { - border-style: solid; - border-width: 1px; - box-sizing: border-box; - overflow: hidden; - position: relative; -} - -.fixedDataTableLayout_header, -.fixedDataTableLayout_hasBottomBorder { - border-bottom-style: solid; - border-bottom-width: 1px; -} - -.fixedDataTableLayout_footer .public_fixedDataTableCell_main { - border-top-style: solid; - border-top-width: 1px; -} - -.fixedDataTableLayout_topShadow, -.fixedDataTableLayout_bottomShadow { - height: 4px; - left: 0; - position: absolute; - right: 0; - z-index: 1; -} - -.fixedDataTableLayout_bottomShadow { - margin-top: -4px; -} - -.fixedDataTableLayout_rowsContainer { - overflow: hidden; - position: relative; -} - -.fixedDataTableLayout_horizontalScrollbar { - bottom: 0; - position: absolute; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableRowLayout - */ - -.fixedDataTableRowLayout_main { - box-sizing: border-box; - overflow: hidden; - position: absolute; - top: 0; -} - -.fixedDataTableRowLayout_body { - left: 0; - position: absolute; - top: 0; -} - -.fixedDataTableRowLayout_fixedColumnsDivider { - -webkit-backface-visibility: hidden; - backface-visibility: hidden; - border-left-style: solid; - border-left-width: 1px; - left: 0; - position: absolute; - top: 0; - width: 0; -} - -.fixedDataTableRowLayout_columnsShadow { - width: 4px; -} - -.fixedDataTableRowLayout_rowWrapper { - position: absolute; - top: 0; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule ScrollbarLayout - */ - -.ScrollbarLayout_main { - box-sizing: border-box; - outline: none; - overflow: hidden; - position: absolute; - -webkit-transition-duration: 250ms; - transition-duration: 250ms; - -webkit-transition-timing-function: ease; - transition-timing-function: ease; - -webkit-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; -} - -.ScrollbarLayout_mainVertical { - bottom: 0; - right: 0; - top: 0; - -webkit-transition-property: background-color width; - transition-property: background-color width; - width: 15px; -} - -.ScrollbarLayout_mainVertical.public_Scrollbar_mainActive, -.ScrollbarLayout_mainVertical:hover { - width: 17px; -} - -.ScrollbarLayout_mainHorizontal { - bottom: 0; - height: 15px; - left: 0; - -webkit-transition-property: background-color height; - transition-property: background-color height; -} - -/* Touching the scroll-track directly makes the scroll-track bolder */ -.ScrollbarLayout_mainHorizontal.public_Scrollbar_mainActive, -.ScrollbarLayout_mainHorizontal:hover { - height: 17px; -} - -.ScrollbarLayout_face { - left: 0; - overflow: hidden; - position: absolute; - z-index: 1; -} - -/** - * This selector renders the "nub" of the scrollface. The nub must - * be rendered as pseudo-element so that it won't receive any UI events then - * we can get the correct `event.offsetX` and `event.offsetY` from the - * scrollface element while dragging it. - */ -.ScrollbarLayout_face:after { - border-radius: 6px; - content: ''; - display: block; - position: absolute; - -webkit-transition: background-color 250ms ease; - transition: background-color 250ms ease; -} - -.ScrollbarLayout_faceHorizontal { - bottom: 0; - left: 0; - top: 0; -} - -.ScrollbarLayout_faceHorizontal:after { - bottom: 4px; - left: 0; - top: 4px; - width: 100%; -} - -.ScrollbarLayout_faceVertical { - left: 0; - right: 0; - top: 0; -} - -.ScrollbarLayout_faceVertical:after { - height: 100%; - left: 4px; - right: 4px; - top: 0; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTable - * - */ - -/** - * Table. - */ -.public_fixedDataTable_main { - border-color: #d3d3d3; -} - -.public_fixedDataTable_header, -.public_fixedDataTable_hasBottomBorder { - border-color: #d3d3d3; -} - -.public_fixedDataTable_header .public_fixedDataTableCell_main { - font-weight: bold; -} - -.public_fixedDataTable_header, -.public_fixedDataTable_header .public_fixedDataTableCell_main { - background-color: #f6f7f8; - background-image: -webkit-linear-gradient(#fff, #efefef); - background-image: linear-gradient(#fff, #efefef); -} - -.public_fixedDataTable_footer .public_fixedDataTableCell_main { - background-color: #f6f7f8; - border-color: #d3d3d3; -} - -.public_fixedDataTable_topShadow { - background: 0 0 url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAECAYAAABP2FU6AAAAF0lEQVR4AWPUkNeSBhHCjJoK2twgFisAFagCCp3pJlAAAAAASUVORK5CYII=) repeat-x; -} - -.public_fixedDataTable_bottomShadow { - background: 0 0 url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAECAYAAABP2FU6AAAAHElEQVQI12MwNjZmZdAT1+Nm0JDWEGZQk1GTBgAWkwIeAEp52AAAAABJRU5ErkJggg==) repeat-x; -} - -.public_fixedDataTable_horizontalScrollbar .public_Scrollbar_mainHorizontal { - background-color: #fff; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableCell - */ - -/** - * Table cell. - */ -.public_fixedDataTableCell_main { - background-color: #fff; - border-color: #d3d3d3; -} - -.public_fixedDataTableCell_highlighted { - background-color: #f4f4f4; -} - -.public_fixedDataTableCell_cellContent { - padding: 8px; -} - -.public_fixedDataTableCell_columnResizerKnob { - background-color: #0284ff; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableColumnResizerLine - * - */ - -/** - * Column resizer line. - */ -.public_fixedDataTableColumnResizerLine_main { - border-color: #0284ff; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule fixedDataTableRow - */ - -/** - * Table row. - */ -.public_fixedDataTableRow_main { - background-color: #fff; -} - -.public_fixedDataTableRow_highlighted, -.public_fixedDataTableRow_highlighted .public_fixedDataTableCell_main { - background-color: #f6f7f8; -} - -.public_fixedDataTableRow_fixedColumnsDivider { - border-color: #d3d3d3; -} - -.public_fixedDataTableRow_columnsShadow { - background: 0 0 url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAABCAYAAAD5PA/NAAAAFklEQVQIHWPSkNeSBmJhTQVtbiDNCgASagIIuJX8OgAAAABJRU5ErkJggg==) repeat-y; -} -/** - * Copyright (c) 2015, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - * @providesModule Scrollbar - * - */ - -/** - * Scrollbars. - */ - -/* Touching the scroll-track directly makes the scroll-track bolder */ -.public_Scrollbar_main.public_Scrollbar_mainActive, -.public_Scrollbar_main:hover { - background-color: rgba(255, 255, 255, 0.8); -} - -.public_Scrollbar_mainOpaque, -.public_Scrollbar_mainOpaque.public_Scrollbar_mainActive, -.public_Scrollbar_mainOpaque:hover { - background-color: #fff; -} - -.public_Scrollbar_face:after { - background-color: #c2c2c2; -} - -.public_Scrollbar_main:hover .public_Scrollbar_face:after, -.public_Scrollbar_mainActive .public_Scrollbar_face:after, -.public_Scrollbar_faceActive:after { - background-color: #7d7d7d; -} diff --git a/caravel/bin/caravel b/caravel/bin/caravel index ccc3f0d85c233..5e2d4203d9d51 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -131,9 +131,10 @@ def refresh_druid(): @manager.command def worker(): - """Starts a Caravel worker for async query load""" + """Starts a Caravel worker for async SQL query execution.""" # celery -A tasks worker --loglevel=info - print("Broker url: ") + print("Starting SQL Celery worker.") + print("Celery broker url: ") print(config.get('CELERY_CONFIG').BROKER_URL) application = celery.current_app._get_current_object() c_worker = celery_worker.worker(app=application) diff --git a/caravel/config.py b/caravel/config.py index d86f61c17947e..f922b9db02aaa 100644 --- a/caravel/config.py +++ b/caravel/config.py @@ -188,20 +188,13 @@ """ # Example: class CeleryConfig(object): - BROKER_URL = 'amqp://guest:guest@localhost:5672//' - ## Broker settings. - BROKER_URL = 'amqp://guest:guest@localhost:5672//' - CELERY_IMPORTS = ('myapp.tasks', ) - CELERY_RESULT_BACKEND = 'db+sqlite:///results.db' - CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} -""" -class CeleryConfig(object): - ## Broker settings. BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' CELERY_IMPORTS = ('caravel.tasks', ) CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite' CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} CELERY_CONFIG = CeleryConfig +""" +CELERY_CONFIG = None try: from caravel_config import * # noqa diff --git a/caravel/migrations/versions/33459b145c15_allow_temp_table.py b/caravel/migrations/versions/33459b145c15_allow_temp_table.py deleted file mode 100644 index 7fab1a13a63e3..0000000000000 --- a/caravel/migrations/versions/33459b145c15_allow_temp_table.py +++ /dev/null @@ -1,23 +0,0 @@ -"""allow_temp_table - -Revision ID: 33459b145c15 -Revises: d8bc074f7aad -Create Date: 2016-06-13 15:54:08.117103 - -""" - -# revision identifiers, used by Alembic. -revision = '33459b145c15' -down_revision = 'd8bc074f7aad' - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - op.add_column( - 'dbs', sa.Column('allow_temp_table', sa.Boolean(), nullable=True)) - - -def downgrade(): - op.drop_column('dbs', 'allow_temp_table') diff --git a/caravel/models.py b/caravel/models.py index 02fb5da177c1b..5c46293d6b809 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -378,7 +378,6 @@ class Database(Model, AuditMixinNullable): sqlalchemy_uri = Column(String(1024)) password = Column(EncryptedType(String(1024), config.get('SECRET_KEY'))) cache_timeout = Column(Integer) - allow_temp_table = Column(Boolean, default=False) extra = Column(Text, default=textwrap.dedent("""\ { "metadata_params": {}, @@ -1702,16 +1701,3 @@ class FavStar(Model): class_name = Column(String(50)) obj_id = Column(Integer) dttm = Column(DateTime, default=func.now()) - - -class Query(Model): - """Used to store the information related to the executed queries.""" - # TODO(@b.kyryliuk): add indexes - __tablename__ = 'query' - id = Column(Integer, primary_key=True) - sql = Column(Text) - user_id = Column(Integer, ForeignKey('ab_user.id')) - start_dttm = Column(DateTime, default=datetime.now, nullable=True) - finish_dttm = Column(DateTime, default=None, nullable=True) - tmp_table_id = Column(Integer) # TODO(@b.kyryliuk): add a foregin key to link the query with the tmp table - diff --git a/caravel/tasks.py b/caravel/tasks.py index 38f55b2511f9d..a93393bdd51fa 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -4,15 +4,9 @@ from sqlalchemy.sql.expression import TextAsFrom import pandas as pd -print str(app.config.get('CELERY_CONFIG').BROKER_URL) celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG')) -# TODO @b.kyryliuk - move it into separate module as it should be used by the celery module -@celery_app.task -def get_sql_results(database_id, sql): - """Gets sql results from a Caravel database connection""" - # TODO @b.kyryliuk handle async - # handle models.Queries (userid, sql, timestamps, status) index on userid, state, start_ddtm +def get_sql_results_sync(database_id, sql): session = db.session() mydb = session.query(models.Database).filter_by(id=database_id).first() @@ -42,3 +36,8 @@ def get_sql_results(database_id, sql): ).format(e.message) session.commit() return content + +@celery_app.task +def get_sql_results_async(database_id, sql): + """Gets sql results from a Caravel database connection""" + return get_sql_results_sync(database_id, sql) diff --git a/caravel/templates/caravel/carapal.html b/caravel/templates/caravel/carapal.html deleted file mode 100644 index fb8f7cba429f3..0000000000000 --- a/caravel/templates/caravel/carapal.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "caravel/basic.html" %} - -{% block tail_js %} - {{ super() }} - -{% endblock %} diff --git a/caravel/templates/caravel/table.html b/caravel/templates/caravel/table.html deleted file mode 100644 index b8326db4c604e..0000000000000 --- a/caravel/templates/caravel/table.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "caravel/basic.html" %} - -{% block tail_js %} - {{ super() }} - -{% endblock %} diff --git a/caravel/views.py b/caravel/views.py index c1c3efe634f1f..da306510e64b3 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -1419,12 +1419,6 @@ def sqlanvil(self): """SQL Editor""" return self.render_template('caravel/sqllab.html') - @has_access - @expose("/table") - def table(self): - """SQL Editor""" - return self.render_template('caravel/table.html') - appbuilder.add_view_no_menu(Caravel) if config['DRUID_IS_ACTIVE']: diff --git a/celery_results.sqlite b/celery_results.sqlite deleted file mode 100644 index d202c5adb61b71b097777073489a4e266b4adfd7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60416 zcmeHwOOxDImR>#dBBgFW#%V!NtKn@9&)Z=%){oBJNDmY!MV4=is%2!^4Ar*y$V`9DEo5{|Nt=e~$3s zwek!6dszG2_VK$1-M{#^-#Px@2d^J~ad7k>kN^1SpC5j4bieKLSN{9I%3pi)-48xE z{Pph?!FV2r@jT8h{36It7jY5neEIq(5AWZ5e82Pf-j5&L@9g}rbMyMOBN9i)|{2o^I_|wns zE0P(OJr2Ut&g1((S2ua``KLd9a=&xa-0ar%TOYsv#=AfL;P4<>M)9-!=^Q)C57tGh zzV>$>%(r&F@y1V%-yoiSsaR%yJ&&KR<7F7{eEHfI%XWTva~Uk+5AmC~e)Ufe-+b4y z4*w9W-+m2wTi`l~M0^!3%BD-K+7;5UK;*Xb#*-X<{`rygK`@cL2<7Z*KDw1@WU%yurSu$Q1@%=1Iv+GX}pPan+ zNxGaQ)Au^*_^UW9IC!$K+|CincUK7hgK3=K)_bSxoLDEe(_LNQij#OcHQ&D-tX8)Ry6MU4;>L%a zd4f!W?e;$GB+HKazRH8(rk&2i&c|3!aTd(+VX|J{d_Tyh`G=kN-#f zEzS2=Sz3VQapV(od|xdQ%Ufs5P9D!EC)uSde3HjmetVowqhRTuUSt>ZWPN%&UByd3 zPuE!(`&qn7!M-31&f@vzU&^24bza$OJyYZhkl4U4QFbO*i%BLOaQ7rsTjZp+5$R(gQKc(~DJS6}Bouls^9Q~`Ke|P-P zAoK3g-J|L8zd8OF$8R6~Ieu{U=Qoc7H;utI)h$e1;U+V^AHz10G zZeaDIZa;D+PH#Bshfe=GUD%7=zB?TBx&zm;p;n<$$8kLB_U&jqjs}kFM8P$>&*?uY1nmbYtkLr5tQ)29`r|1Z{&vFTXf;17xeA16Vsw_Vc^)^U^p0coxyNCnT*`g z#E##j3*%7`Il-vc9eNX(2(LHnj)Fnd^_W_p2SN8wP{%?CI z{~y`^>`rGtKk|A_{{Nd~`@e$MJUBM~e=N|jZT`PyojAkMz#Bb(aJ2FNC7Spf_5V$v z;`OXy|H}WrJT&<>`u|t*-=Lo!yk_KI;)=D&zuiCS_xc03|NO!6M*bxV<(rm&QdA?$ z9oSd$e zKYpVW|3s1{{$|Br>nX42^?O(1|Lqa~A8Z%?{?g?CTWbH`G73OaCu&dtgF9C5jy*W( z^@l^R|NPOLr2?2rG|D%t0F0*ctl^d8{{l?HtDE@yOOyZZ!D08{zj71$%75p+3L$`fOjQ_dy z_4xlkJUIN}!JobI(q4Oj2iJ|@OHe0mf)6a6HR|^U*7KiTD+NE5Xp(PM@QsFo>AwQ~ zeX9kZ+yAdc{*Mm+_~7VE{1w*(=#JsWljx({#Z3Q}qH#Bh&k`U>`44~-o+Znve;Qvj{Itm801dB8 zYC+v4xeBr(Av|*R1ANNVs&RL-?2n3Z8eN!g>W{dJx@zw}J=}sH!v%5gc)&>`U*dVc z{@v^9E#0|yAW^~9Tl(zkE!AIqNN*6(OEUz4(I!`^TRG5_n>m(?AkyY>KgGFz55 zUvxKEMp>F*4N(99`&sO-EdR`|uEHlnVV9IUtgdRK<@je#byYhpuUB1l2`#s}s+E@I z37*lFJ)`GUH{4!UyaG zzqnY%0g0@SNe+GpjSy77-g=XEj>h^PhWc`i<1rMWgkSw?D|0NXwkvmT zkZM%_({+-8co?Q@(Mv^#NOj{KlEpf%e%87ur(fIN+Z3krB!Yohpj88VBY=4m(6b@` zzttaAS8mhP)K*zMi3{3P)9S+3CEC4Kf%s~*&S(BIEs{wR2FCQ1#<5k03|imAJe~UM zWwL~AUXxs52Q7oMWGW>+Y?ZNPd|ryyWS*Y;5@qL%YFBOZab^4KY>Ign7ycs7bG#w} zBQ2{{d9ECPHA{=s&u2jl}?L>v^2}!8D6wp`LQxR!y44(VAor zgFqnRkUOZ45bK4MBcRoqdbYzQ${#(-hVLe(Qj zbBQH#J+4O7vgW~h3ERBqlAHnGG+CCQSI&>nv6;-3)m+uKLhFWcd$nz4cK5R+U4X&_ z0Rc2-Qe0Fos+|rug=4bX5N;?AY!H&a^~#6hh3(N#UAuETU5{~poTjHF6NskPq}lMy zVUpuz?kCoL_DaU<91V#%90ApZ=WD+jjDXnq0md{H0!G+m|Q{ip$e5(L$b11PgYCwfCTR0LVcz$MniziXs2sgDnZ5 z{A*XXFOLg0dM4q#kv5ALH1H|&lcnAji*#Ac=1|Dw2TLT$ace9xt1YSL7t7Emv%Eki z6dNP=&yx6D!h#XAP|82{)5R2bN zhkO$i>FGr(AP?wmbyb}_9P0u-)*?x@+Z4enA6L}zsU?@uYuRVY{fcOSBGPQ+(1^B` z)1o%2NIpp=gscXG0WAi7enE^$&zDHsoWazu5vp1;Mp+FSr;TJ3PXhF*BMuABpynwx zAz-G(vnC8*k{q7~?npV2ZQ{+d;KZP^| z7%ZvM5cmfJ5v3_79)=?CUqG~cRX8|@IrJ%~j-8zQvtltvWH-MMtQh(7>MEn&Qsbz? zpDGD|?Hl#MD;2>=t{Fce%cf zV&nj@xlyzVi_6lgkpI6$g{gW0!0EH?h{zKYQjil^LDi8IA^o%jAJ&U8rE&L4Z4a0t ziI7GO3q%T#7Z?EltY&|(kDBeV#r@VO$yI86f|A~h6go3v>A?R7{aauX)@|kpJJNQV*1lsUI{dA-VQ+F57yN zn@Bc_>(s8QKaP5o9QCC+5|T)A!nIeWfLw3$MT*>w>_s@dR$JSP?YG#2Q_f|2>P2^` zF`!keuoOe>2~_39HA&(*9OpF(PLg%my#7_Afc;5;7r{HRhfEVq2)m>z zuhTWvHO(tAs1tZ|V?q$@2TuSZv098)MMzg+iY4J$0I#F+k7!$39NIBitE?S@IZB94 zvtTuoDKFP%_HL@cNQn-xFnY9rV~is|j*Nt-lOKhf;i7s$6AZ!h{mW>ofwKoK$F$%c4O24)m?U4%M;eQ~&h058Nxb3k$ zQ4zO$nj@4LMb$ORU-~v^YHoi9+eV5QYJ2kx)us5^+s>xC-gs@Iyu zH0v0O(uD4kx7xpK)h=`zVAaKi8s&>rV@NZXRlXNhcu8f#wwbus9>Uw957}B_pfiwD zGU_8%L#h$)8%!t42LSydTMK;xrN0oc-o9)<+5-qv@C#siwwR;HLjHe8gVWpIs?8jb z+LE`W$Unz`;~M)~cZ$X9c{QRt;v=T&^6XVb)#;Jt-w_!x>0yI%MNSct-Rrr~uu9Oi zibe82_w7N?kAjOF22tw(!8^2g#6io~H}SS_0)!qmT(*55nY#6w)vdlw-~UCNLHDBG z0;wb!HOfn{erHsB^+qSg^A*njLedEs8(kSuP8R`sBouijZdRk1#d9zLl|oZ<5m0n_ z%rzs}tikZ&Unp$-9x<}J1U5+s^X#rRT*@iJ&I6o8A@c;Z9PUzTTI5yk=UKTtd{Cr9 zQi(KU9PDLaJYfsuAII zJ5g=GZX>5l$!1-!aLXb?MN05aEMB713#8mF_F&z~Qm%x6rrRJe#LK;LQ!cNds>Mlt z#|D!}>!>-n>XGu32?AK9)|7ClLbCW_ZEpJLCM&|Lqap}=Bh_bRioxYIJN3Yu{8F+! zOVF=CE^5AzykDCGfww8nq0Esd24YeKaBz5rYMUOpLR5;?k1yCQ4u^y9Qp}OOnL^bP zDqQC4Q2zh#ZSwy{Y13=jaI(Uqi1L@ts98Vcvp8F!4TBy#CXX1sZwa-rh^V?ef2*Fo zPBo=Eq@gNdC(B$Zisu;}UWsTJN{n`oP7z3UkTYbC5^(i&W(J%uylO!I^Qo?#NtVZJ zxvvVR7m#9uLO{|PFl`2sFz`A!lPX1vCsROX zQ~%rdK`onQeCyG`C0d+p<^&i3vQiN|2<$`=n#5|@&jx_*ZxxrR z-A3>NTyEi$1^9mkTa1%eROKzBA}2gp2|1^_NNYioEu>F^i?u;^RTtn>a{%n|h#Cl{ z6=552s0&!7Qab`d+*WSWS=E-?x*AL^t8P_36vaRWPgS{W3jU5ayON~OTZt;y{5*!1 zQs~mA-A1c0tkVl}c>7SzbSs~46;sb1p&22v7yju|v_CAj#v=3ZDe?hT2W!QfRt{R+ z*##)GMx=I+ONekSxkaQFHMJ74c|upMWS(G$3ZBH_4wL%*73BZ#!-J(@Qb1-0Y3(q7@F3MtHJu5UCYOuvnYpo-ZRRMBVUg zFcJNFT28r5A=0wWP`APTz;6?tNOy6n7EaMy#;G?Id#l%sTDI!kgN7}wdUY8-HS0v4 zLneNKR@o3q!6JG(uw!CiJU2bkxqvpWm?+vVViba?WhuOC9R!G!k;kieh90bF%|F<$ zL6VoV6j5@rOTbd;Y)hUQ2hLTTsa}rh)NWv8b#>D_g8bi5J^l`Fqo8$~$_8ZmES4ym zW<9kE*7g5E5c&DN-7 zAxpp>qAFknj_#_i(mQmEsM)GnuvDFXV+Bl>bo#iwqWfaPlGYBFW121_Oyx9LNmLmT%=M@s=*~k2Q&rjGT)WA>h4m?<0<7RlwPT} zG2`2YXEjZx_c|KyrVvVWFCgR%$teOXSvn!Y;RT|0dVXEmwy%))Ezx__4x7pDOLT?Z zXC=>hSxxQrcdDsc8>4_IU?Ii@RaccfguPW|T4%SEO3>bvAQcFcgB-OteeV6RpqCCft#}36c zz%!Jw#()K&%Vny|=7#%4RAnhqzUR|Dc0kY!+6F z>r5gFr(zmos7s`}ZGtF5^$VqLoqr?iS*@g%GzWEpGRy{WPkK|(6q6X#0na0<6gb>TGSmQRT(4-yQGa4)K)tg0H6UPtGv4r`@l1dbQjqDj#= z`Ryv58l_ti5#;}eEyC~eaXW^rp58_hrIJd59cs1iP(5hJH16pkZAjuHz=A#uCL~le zK+2c-csTa5T8vSGT7fj&hVnxg`vPl*;d#}s;Zw6R(o)n{k>gr)yMuAD>q?JB0TVq%RiW8AL5^%xcIbgF?MGERhBVUmc zr4fGuigttL(l2eR{kv|p+AWsDODAS|rMg!Ye0p9r#1!saEv#vM_cl+7mSo|)mKKoQ zS2PaOXj#E?^I1YQk_LB4c}#SvVJxkHpvQ5~HHrrMt;a^rgLA!YAph@O?*H@oHZ`KE zVtk?PHu$hJTYBrV=sTqC(g|FRS^WpT8lp+v4Ec{xyKaBHA1G7aBUtP3i|1}F|h~LGKYK~+om#EY{`ek4;pGn7%S*=?c-4swtC6c^a zKRz`(A^;axS8Y8dAs%QXZTT|FCX!22z-214bxUvi`~GSUtnf2q-zCn6&4&OOP8#iL z{K*M2xgxX%4mfUMfFTIi?imdkl~k;(R=>{7+PlnsZh1paLpnmcG{9dVK7@OZyfAcw z$oXhx-$uJnkdhMmZ|-)H6Y_tHMttkeSXE2fE7rCe+`$XPazNgY<3i&AN$pUQzItV? zJKN*=r>*89kt*d-hfQv!j?yhrR)S?=oPBkb7C0ig0UVj6iW*xri?-scmJMR@fl`Vh z!B$bZ1~9FK8Y??ZR#8C6;`987Pzuvb z874o0N|WdtmxJLzXC0~{_kN1O?lVsjZ%|EIBnM;?Q~%EgSmS(-{Ge#hx!f%VNhq7u zoC-t0ALK^1;b~J_xOyD|_Bp&T2*PF7X0z)=9OQ`*kTQju2|DM1P(;abkBZHxdH|zf zK~E38))NvS#wD5yb~cpYWE@^o!h9EKD?-Ep)=7nwa*^aKs@Z+jI`F9_Jm8@u=<166 z7SvkTZ1LLt6vzFbvzm^v$P!UX-{LMf-J~43wpRoEux3mQcd4hc>RK;7nsS%nXKJt= zEQo*~4tFZC)O-dI1XAs!dQrWiR)9UqHZD)&lr-x=!K{!tgAVTi?OXNl$&?FA z--9$^X=q$S9G;HTXMz>l95_g1YAN~sw5Zcy!aa=Rz=>PbH%Xo%-!`g@w2#@FrcwjXMm50pA3JccbRQA~IcEc+tVzp)h0$|C`adLTj5X16K zBAHTp=3R@nw`13`_eOV6tF&tTpaiHYfSScgWYOXi2C+H>wBhN6DWpoc&qH=25xJRU z0@GXDgk(z=IAKdb+T)PTQrPH)=!0;KD!+g`Nz!A6U^P*P{C9UeKbucHS9N(ce+GjU zScP&Q#c4NqTXQ4_XkIH!kjdZ3YCSd|eGMn1K=~NhJHUyk6t8l`bU+OBydE=(PP`^Y z>@dM?@SLa(p^M@~w@AAPw7W#RD~#1Gj)PfYx;6F{^hSB+1yz6rYvXu1s>E(4E<-SN z2WV~n(3Z0zaiJVu@WM@K#t-#a`D2ukPPa6GVBKy*(#RJwqh3(pqMT z9V^khc0<6W$*55jdS+TZwJ2bk%VBA&8cn4I_atd?P)A7;Ejko|yxdB8V>ig+Fg=WW zQ1J&m3G#Lk7^icX=OoWX$4Oc)Pi%lL&f~)nZm5#UBzflJ(B2jGw9(Lwb&HTe{(Huk(t6V3z}zOvV)p{Fwd7UnVr^b8znrR!1-xY2MAbO>_5BI?w_4u1 z-H<|Rcuj_=>eH%IZlUUM*dLav)nQqZH}3JK9~V)lfHLo11mPJD4ODg!c{!x~Dd~%1 zidMPOh7GsD!v*CE$OfUJ;__aT=xmNtq~%^h)G$fdlxEjAj&Xqv{c?0Za!`>1L<(~l zPDAPmBvA|W&(b+gVBQ#HrO{0@gvg@Vyyt+pL2)PG}y#iH>Pi@?x4D1MoA5dQ7%$6i&T|Vdm6b+!6 z=IcXZ~p%BXS}wK`X|; z#ziORLpZ_2=B<<9<~GC!H9>VKtQFurp=%r4Td-uCL$6B8Xpp1GL_oyl^GQLZ+>_=I zF=2wZ(KA5P2c?5)4=P5MDnYWghGYQNMq_%(wM68#r2~YG@(yo+z!Ot_23;$$9+U!s zR|77@F1OuK86lzzO5cW9w4Rp%`FG4AUOE^dmAlW_dYgCotHrDPDBBPgNW;^PrF6H< z`}P_#`w24Qp;?fNI7k zi<=%s4YRDar1&09SuE^-s39G*CCG5#gbx@{N`KS&C&rSB92Tdciup|Ym*WKGg-t1I zlf~QG0it_Yyq5{@Liw;uctmBVq{)H^c4)%}h5WmQCh9w9@)9Q>H$`g$3IO2n4O9+$ zF+tuRAbCkb#%-uMJu?YMN?~Y|N9S`1E6u!b0c+F=%w2jCx6LqNW6ywzku=%Z99C0by2?qb zbT1*K_el(KC$dg=qjC2o>+k`(po^mmApc&j2JupqK@8TzAcF-wq%0C0KHzY`q9(39 zA_9r%gDQ~S6UYPa=&li?@sed*28JACy#k6BpV+}Uh`_9Ay$qEBp?FLRH0Yw)`#b`zT&N`wEZ_E2~h)$38(~GH<#4L zAki0Y)ANE5&Dp2;K~1YLovLcg%6_RMn|Ivgzz1%V*LF%n871Dd&2tqEL?QnJl|M8O zF~Od%~2SeDC+T8f;lnxktp#n~&1>u5#MF?=+#=HLJ)fT=b^g9R+*MSnwB$#C`f z;-$Jxyv#ANgWMcC&YRo`)!HXmo!z7hRnZzxiIgRkud!dHKnxBU%XJBSLk;pj#3+M&oYspL+k|1-{IAp1jJfQa8r0QrUV0A#k}i$8d(hssN}*z<8U_ z&2J(Omczo+86oAV8D#^Ij71L5ps3rJl#|w+*pVBC36^g0Xi?ELV-Zyl1Ay6*CjMtS zTO*)q*Zl%x@u}F0hr!I-YEDIM^-govk@d>6q1pd{>$ ztrHnhi|6V0x`l8u7O&;&yFmU&zIhnAUg4cehvj7=O80@*hsFz*Y4LgvT4>=e=*KrcsW^9TH-va=l*YBk zpvE)Qeb0^ZgZ$rtx@_C@4sVCbsq#LdIEs#9na7y>D|lQErlLX4gqkVaFE>lNWH~Me z{Sh{#BtaWfbR3G!D=P)2WmKm@A@(nD)_|F(BdL8itPJfXS_R4#tIY$=0%Xb#Rg`FH z5(cn_K(kXZRls?h1FrOliPDJcHSUqNcevx!L@{3`tCeh^kIg$^&9G&-L-3%H`Il%w zms>fFyumCqa8bq1hLz;!3YdqCLv@vQD+px)_e33bN%?ZX$?|g=wzkfyOYy1Mia0z! zM50tib6SJKyC|g=)i}T>_nHRyU#HHBP6%EmN?YTCglGBXGrK#F)A-B zk!FWu?SONk*c=UqXks&6jskwT9ICZLi8P^v5NmV-9hxLaKreiR@!D(Jr^CnVQSm{| z5jD9aRjnr37L9v!d9~|2R@>WDxK;P2RH|JrgXgmg<&cv>H=i#og=pKrrvSPY+Khb=vLqyY`@C4enUp9m_iLyuRbP-+}xe9{l?Q`tKF{^N0W8 zd+%I3c=G+{|N6J@9ZcLf>f3?Ub*#Yaxa`nKc(}PbHW|ppu-hELM$GXa}kTILj~>l;~V6^bdevOF7I?Q%18f5U9D>} zv%9J%d{!OKIJ--!8L~U4>O3a%BfM*6Z}4w*e0n8OEK{0&%1AS=vmU#mhp_#p)hVC??KVp?~RS0U|@_4gqbCtG^c7`6()J z;$$9ocgcL1wbmVIpIVf~a0K3HM(2U!Rlz4o=l*Ge%mhjan5;%L5d^Fx5vk4dWWgK> YQ5tOnAMre%qb_O^KTSM^y#KZT3pP@a>i_@% diff --git a/celerydb.sqlite b/celerydb.sqlite deleted file mode 100644 index 56b518a3d96dc1e8af5c5195b719eaa9b830a77a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19456 zcmeHO$#3IWdY4R>m#XSaCX=8rhcpEDp>sv=(Y< zTW(8g%%Ov%4q5(-9COJn*C5Ct%S;YQfC(_kAvq)nW-yomvH(ede3V>{?Q(T0Y0T7O zJxEW|dwknR{oeO|zwhy|UhPi8Of)(fc#{lGt*6rI)VDI3R4Vm8yuSwT=V=|jB=6@b z{rt-pe7>I&fBXmU3ICqjNd0C?_!Z#~*S?pYrq|)e3u(Hs`6&B1{rN9-hdi8ir$_zK zz&|@0gr}$8adRB@> zx|Bqw0J3b94LvCxd$V}t@k~Q+L%$b&FM0@1PP?OF=FrGit1$Sr%1_S1v+(GDW|cI^ zAL#jyGrP}6E}NNM^kD`+d9tXinTHK4oAk3&{P@OAU-Y(l0Smp{4xXn*ZH!&}n(=6-Qxl(cFQgfOs9_O-jOPx+YI%9vpBahfp zo5cNUK38a9yJqK)Kl%7DchKx>b~B&Lw+?e7r)`eBmNaaWNgUeR*ey(^Imeym;%3ny zolk>-HO(Q(&bQ2tUz&EEmU$9Tz1)bKWw%|KIfG))gZbo}lAWU#j6IpVZEKqEOLle; z_k;Sxt`zL9F*m5$Xr5Oaxt`f@CT5|2?#vK#OJ%9ruw$c8mfV&zGv>Lj(bf%+!3_GD zd96eSxSB0CbzUmYJ*wAi3-_FMrNamIX?3_9vpV20FKY9kHH{Lv@zxZnI%^Sqmz>`Ui1ZIL{caDlq!;hL=;G}$QTMlmHV0~g@|ZjL@4rc(()u6 z$Frl!C~1R~2&-82MIJ`JNIht+N+A<9rV_V3qwX+D8h!>d?T${m zljJW5RL9QA=xhRWJnDywKjl*OCZmPZpyLgX7wf+$O+<6q;b6p*I&!sa;w=`7Jnlz4 zVj_-wA~JyFFFb(ktBFPb^V^VpC2{b7ZWFTiHXwVrfON@Dt&O!aoWBApD*1H^N^De_L*YPK(_dxR z3MT8fR)bh5K^kSkt^YLrC{Gj4 zAV(|EPi5@3&1peD&DwPf0atMXT!4K5TtqM^61R1Jwj9$qbBT@Gz(e>&m~-32{(%Ma z1m2=;js2E=3LL-~=JmAEmrKB>#LzC`0_@yiLV-&d0#{%~ZT*lRvX2sH_qF=}JP2i# zU?c{S?~5#=TvS8gk0PHU9LchV$qW9!M%6G#NCKpVRHVKfi9QQd$fzv)h)J08WMP-? z`p?q;_X)-TawPg#4n&3&E_#$GqOTy?qgp@$jjZthg?rkwl<-pgU)oo1DE=GA=SIu! zIrBc?ztVLN5i)wbTW!Q0z&zj|@_Bjc^s>Yll|Gm-pmsq3_MFYSG2o}U;;3YN2VM8OZ)`&&EroJlvzmrPA>#a}E(E1`a z@c0&ee(6qqX*_oleSR4m$d8V#xbn1dnCrW3ckB>50-ZYdTKo)j-VpFs}?^)dVg-F=`}?Zf)aEi_rJ zVRveWxbHS9ooXW~3nos_VFr|$jY@3Ht*+bBr*5w^Efkvvz~Q4{5XYdG&BODYojX|Y z{L@0*1SDAVz_et2IWh zorO6%ZQTMJrQ}f#wgjIC7Q|jG6IYKvdj0yJa7oo9PXjKWcz}Nnz>!rfa*2eL`Jsm+ z^{v7`^+?DlQ$(&Pp#KBHMPK!)sL7h51yn(VN_P(b_g;ekD{RXgZdPfH^B5X)y0Qo;U*oj{@}q5=5!Ar5ZhKT=c$aUzMT z4~_$6OolT91SkYQk~Q$L5mOd>#%sYpoTX5ah9fZwVTR13il{}v1V}-oa0N@mqwLP% zfAb~yzY?o?C-6ToN_yhg&%y7XyMs!v+ORstygYFl$Fv4z=DgE&2aaSGzz!JLEBr^u z0~a5(K8<`Zmw_r1JyiizgF=rYjcE~CoaMQC{wIvUHBe<}qk^AJkrIl&M|}~F{U{hI zG88QKjMswyKnp^O2ossAP?1p#+Z60@Vi@3vf`vjo&%bl{-?|0=Nf`M`DED2!e|_o@ zhdMp8XXY{3_c7qVJ~LVl*#DJ&at6?xmnrxFeWTDv&U}Uc9SmyGrvVegPy_rEP@Ed! za0n1cG+EeQIm=PtzM?!5hvLMDTz zM54kl9G7Jf)$?I15M{!BpDUD6fD6R(uIhgb{xwk(V$Ju!2*3>J0>(hkGEZTVD&q*f zM*rXUSjZvjA2JpAf0+aIN5HiS1D~t1hbW2e9{%sY6#th8!QKV@TOGILAg5s^`oH7! zEa3mGjxz`TzoDOl|4-b4IWXo~h!MCwWA3i-@4-=dkAeR8dXACl*VPG{Pjrytt4vieWs{V)Pe-@7by~Pp0XY9cO``^44 z`%7&9D`!N%H1_-dx7eQ<@m~3-5&x5%fgu?s;Q#YzYLNOA^gpS!nh^hM&YT4&0I`44 zBya`*;({yvzoP+#2Tv~{5zg#jP?s9`^1zWLbO4-~lLJEd3+I2{X#EdHz7&xFi+H)A zaU^P>_!CdShj0eWEKaKdm zH84wd-|SU75bK*8EgdiQKj6Pr1e*YzLmUv*8pkuR4SI0xO8-Y-EX%~hV8%lj5A?qz zdLEbqNK=@m5;)h#Uf}=UX#9s##Ib}VF;XH;WEzFwgbJ<=*0eAR;IT!_`Twr}{~Ohe B;TQk_ From eb0d5a2af203412df39f5680fefa2a875ae5a4ab Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Tue, 12 Jul 2016 17:50:00 -0700 Subject: [PATCH 06/14] Refactor SQL execution to use the celery if configured. --- caravel/bin/caravel | 10 ++- caravel/tasks.py | 45 ++++++++------ caravel/viz.py | 2 +- run_tests.sh | 2 + tests/caravel_test_config.py | 8 +++ tests/celery_tests.py | 114 +++++++++++++++++++++++++++++++++++ tests/core_tests.py | 112 +++++++++++++++++++++++++--------- 7 files changed, 245 insertions(+), 48 deletions(-) create mode 100644 tests/celery_tests.py diff --git a/caravel/bin/caravel b/caravel/bin/caravel index 5e2d4203d9d51..5db60e823e1dc 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -9,7 +9,6 @@ import celery from celery.bin import worker as celery_worker from datetime import datetime from subprocess import Popen -import textwrap from flask_migrate import MigrateCommand from flask_script import Manager @@ -134,8 +133,10 @@ def worker(): """Starts a Caravel worker for async SQL query execution.""" # celery -A tasks worker --loglevel=info print("Starting SQL Celery worker.") - print("Celery broker url: ") - print(config.get('CELERY_CONFIG').BROKER_URL) + if config.get('CELERY_CONFIG'): + print("Celery broker url: ") + print(config.get('CELERY_CONFIG').BROKER_URL) + application = celery.current_app._get_current_object() c_worker = celery_worker.worker(app=application) options = { @@ -146,5 +147,8 @@ def worker(): c_worker.run(**options) +if config.get('CELERY_CONFIG'): + print(config.get('CELERY_CONFIG').BROKER_URL) + if __name__ == "__main__": manager.run() diff --git a/caravel/tasks.py b/caravel/tasks.py index a93393bdd51fa..0ea5ab0569a1e 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -1,18 +1,36 @@ import celery from caravel import db, models, app +import json from sqlalchemy import select, text from sqlalchemy.sql.expression import TextAsFrom import pandas as pd celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG')) -def get_sql_results_sync(database_id, sql): + +@celery_app.task +def get_sql_results(database_id, sql): session = db.session() mydb = session.query(models.Database).filter_by(id=database_id).first() + return get_sql_results_internal(sql, session, mydb) + + +# TODO(b.kyryliuk): merge the changes made in the carapal first +# before merging this PR. +def get_sql_results_internal(sql, session, mydb): + """Get the SQL query resulst from the give session and db connection. + Attributes: + sql (string): SQL query that will be executed + session (caravel.db.session()): DB session + mydb (sqlalchemy.orm.query.Query): source of all SELECT statements + generated by the ORM + Returns: + string: table in the html format. + """ content = "" + eng = mydb.get_sqla_engine() if mydb: - eng = mydb.get_sqla_engine() if app.config.get('SQL_MAX_ROW'): sql = sql.strip().strip(';') qry = ( @@ -23,21 +41,14 @@ def get_sql_results_sync(database_id, sql): sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) try: df = pd.read_sql_query(sql=sql, con=eng) - content = df.to_html( - index=False, - na_rep='', - classes=( - "dataframe table table-striped table-bordered " - "table-condensed sql_results").split(' ')) + # TODO(b.kyryliuk): refactore the output to be json instead of html + data = { + 'columns': [c for c in df.columns], + 'data': df.to_dict(orient='records'), + } + return json.dumps(data, allow_nan=False) except Exception as e: - content = ( - '
' - "{}
" - ).format(e.message) + content = json.dumps({'msg': e.message}) session.commit() - return content -@celery_app.task -def get_sql_results_async(database_id, sql): - """Gets sql results from a Caravel database connection""" - return get_sql_results_sync(database_id, sql) + return content diff --git a/caravel/viz.py b/caravel/viz.py index 13449b720dd3c..d85fc83ba6ba9 100755 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -284,7 +284,7 @@ def get_json(self): cached_data = cached_data.decode('utf-8') payload = json.loads(cached_data) except Exception as e: - logging.error("Error reading cache") + logging.error("Error reading cache: " + str(e)) payload = None logging.info("Serving from cache") diff --git a/run_tests.sh b/run_tests.sh index 37ea9249bbbff..95de796a98748 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash echo $DB rm /tmp/caravel_unittests.db +rm /tmp/celerydb.sqlite +rm /tmp/celery_results.sqlite rm -f .coverage export CARAVEL_CONFIG=tests.caravel_test_config set -e diff --git a/tests/caravel_test_config.py b/tests/caravel_test_config.py index a7de4569499e7..e5d77991a186b 100644 --- a/tests/caravel_test_config.py +++ b/tests/caravel_test_config.py @@ -10,3 +10,11 @@ if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ: SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI') + +class CeleryConfig(object): + BROKER_URL = 'sqla+sqlite:////tmp/celerydb.sqlite' + CELERY_IMPORTS = ('caravel.tasks', ) + CELERY_RESULT_BACKEND = 'db+sqlite:////tmp/celery_results.sqlite' + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '1/s'}} + CONCURRENCY = 1 +CELERY_CONFIG = CeleryConfig diff --git a/tests/celery_tests.py b/tests/celery_tests.py new file mode 100644 index 0000000000000..187a3074949b9 --- /dev/null +++ b/tests/celery_tests.py @@ -0,0 +1,114 @@ +"""Unit tests for Caravel Celery worker""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +import imp +import subprocess +import os +os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' + +import time +import unittest +import caravel +from caravel import app, utils, appbuilder, tasks + + +class CeleryConfig(object): + BROKER_URL = 'sqla+sqlite:////tmp/celerydb.sqlite' + CELERY_IMPORTS = ('caravel.tasks',) + CELERY_RESULT_BACKEND = 'db+sqlite:////tmp/celery_results.sqlite' + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} +app.config['CELERY_CONFIG'] = CeleryConfig + +BASE_DIR = app.config.get("BASE_DIR") +cli = imp.load_source('cli', BASE_DIR + "/bin/caravel") + + +class CeleryTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(CeleryTestCase, self).__init__(*args, **kwargs) + self.client = app.test_client() + utils.init(caravel) + admin = appbuilder.sm.find_user('admin') + if not admin: + appbuilder.sm.add_user( + 'admin', 'admin', ' user', 'admin@fab.org', + appbuilder.sm.find_role('Admin'), + password='general') + utils.init(caravel) + + @classmethod + def setUpClass(cls): + worker_command = BASE_DIR + "/bin/caravel worker" + subprocess.Popen( + worker_command, shell=True, stdout=subprocess.PIPE) + cli.load_examples(load_test_data=True) + + + @classmethod + def tearDownClass(cls): + subprocess.call( + "ps auxww | grep 'caravel worker' | awk '{print $2}' | " + "xargs kill -9", + shell=True + ) + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_run_async_query_delay_get(self): + # DB #0 doesn't exist. + celery_task = tasks.get_sql_results.delay(0, "SELECT * FROM dontexist") + self.assertRaises(AttributeError, celery_task.get) + + result1 = tasks.get_sql_results.delay( + 1, "SELECT * FROM dontexist").get() + expected_result1 = ( + '{"msg": "(sqlite3.OperationalError) no such table: dontexist"}') + self.assertEqual(expected_result1, result1) + + result2 = tasks.get_sql_results.delay( + 1, "SELECT * FROM ab_permission WHERE name='can_select_star'").get() + expected_result2 = ( + '{"data": [{"id": 20, "name": "can_select_star"}], "columns":' + ' ["id", "name"]}') + self.assertEqual(expected_result2, result2) + + result3 = tasks.get_sql_results.delay( + 1, "SELECT * FROM ab_permission WHERE id=666").get() + expected_result3 = '{"data": [], "columns": ["id", "name"]}' + self.assertEqual(expected_result3, result3) + + def test_run_async_query_delay(self): + celery_task1 = tasks.get_sql_results.delay(0, "SELECT * FROM dontexist") + celery_task2 = tasks.get_sql_results.delay( + 1, "SELECT * FROM dontexist") + celery_task3 = tasks.get_sql_results.delay( + 1, "SELECT * FROM ab_permission WHERE name='can_select_star'") + celery_task4 = tasks.get_sql_results.delay( + 1, "SELECT * FROM ab_permission WHERE id=666") + + time.sleep(1) + + # DB #0 doesn't exist. + self.assertRaises(AttributeError, celery_task1.get) + + expected_result2 = ( + '{"msg": "(sqlite3.OperationalError) no such table: dontexist"}') + self.assertEqual(expected_result2, celery_task2.get()) + + expected_result3 = ( + '{"data": [{"id": 20, "name": "can_select_star"}], "columns":' + ' ["id", "name"]}') + self.assertEqual(expected_result3, celery_task3.get()) + + expected_result4 = '{"data": [], "columns": ["id", "name"]}' + self.assertEqual(expected_result4, celery_task4.get()) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/core_tests.py b/tests/core_tests.py index d24d5f1207926..2a215ffb2c088 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -8,15 +8,18 @@ import doctest import json import imp +import mock import os +import sqlalchemy import unittest + from mock import Mock, patch from flask import escape from flask_appbuilder.security.sqla import models as ab_models import caravel -from caravel import app, db, models, utils, appbuilder +from caravel import app, db, models, utils, appbuilder, tasks from caravel.models import DruidCluster, DruidDatasource os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' @@ -26,6 +29,14 @@ app.config['SECRET_KEY'] = 'thisismyscretkey' app.config['WTF_CSRF_ENABLED'] = False app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True + +class CeleryConfig(object): + BROKER_URL = 'sqla+sqlite:////tmp/celerydb.sqlite' + CELERY_IMPORTS = ('caravel.tasks',) + CELERY_RESULT_BACKEND = 'db+sqlite:////tmp/celery_results.sqlite' + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} +app.config['CELERY_CONFIG'] = CeleryConfig + BASE_DIR = app.config.get("BASE_DIR") cli = imp.load_source('cli', BASE_DIR + "/bin/caravel") @@ -41,7 +52,7 @@ def __init__(self, *args, **kwargs): admin = appbuilder.sm.find_user('admin') if not admin: appbuilder.sm.add_user( - 'admin', 'admin',' user', 'admin@fab.org', + 'admin', 'admin', ' user', 'admin@fab.org', appbuilder.sm.find_role('Admin'), password='general') @@ -80,7 +91,7 @@ def setup_public_access_for_dashboard(self, table_name): public_role = appbuilder.sm.find_role('Public') perms = db.session.query(ab_models.PermissionView).all() for perm in perms: - if ( perm.permission.name == 'datasource_access' and + if (perm.permission.name == 'datasource_access' and perm.view_menu and table_name in perm.view_menu.name): appbuilder.sm.add_permission_role(public_role, perm) @@ -88,7 +99,7 @@ def revoke_public_access(self, table_name): public_role = appbuilder.sm.find_role('Public') perms = db.session.query(ab_models.PermissionView).all() for perm in perms: - if ( perm.permission.name == 'datasource_access' and + if (perm.permission.name == 'datasource_access' and perm.view_menu and table_name in perm.view_menu.name): appbuilder.sm.del_permission_role(public_role, perm) @@ -96,15 +107,15 @@ def revoke_public_access(self, table_name): class CoreTests(CaravelTestCase): def __init__(self, *args, **kwargs): - # Load examples first, so that we setup proper permission-view relations - # for all example data sources. + # Load examples first, so that we setup proper permission-view + # relations for all example data sources. super(CoreTests, self).__init__(*args, **kwargs) @classmethod def setUpClass(cls): cli.load_examples(load_test_data=True) utils.init(caravel) - cls.table_ids = {tbl.table_name: tbl.id for tbl in ( + cls.table_ids = {tbl.table_name: tbl.id for tbl in ( db.session .query(models.SqlaTable) .all() @@ -116,6 +127,28 @@ def setUp(self): def tearDown(self): pass + @mock.patch('caravel.db') + @mock.patch('pandas.read_sql_query') + def test_celery_sql_execution( + self, caravel_db_mock, unused_pd_read_sql_query_mock): + mydb = mock.MagicMock() + sqlite_connection = sqlalchemy.create_engine( + "sqlite:///tmp/caravel_unittests.db") + mydb.get_sqla_engine = mock.MagicMock(return_value=sqlite_connection) + tasks.get_sql_results_internal( + "SELECT * FROM TABLE", mock.MagicMock(), mydb) + caravel_db_mock.assert_called_once_with( + con=sqlite_connection, + sql="SELECT * \nFROM (SELECT * FROM TABLE) AS inner_qry\n" + " LIMIT 1000 OFFSET 0") + + caravel_db_mock.reset_mock() + app.config['SQL_MAX_ROW'] = None + tasks.get_sql_results_internal( + "SELECT * FROM TABLE", mock.MagicMock(), mydb) + caravel_db_mock.assert_called_once_with( + con=sqlite_connection, sql="SELECT * FROM TABLE") + def test_save_slice(self): self.login(username='admin') @@ -127,7 +160,12 @@ def test_save_slice(self): copy_name = "Test Sankey Save" tbl_id = self.table_ids.get('energy_usage') - url = "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&collapsed_fieldsets=&action={}&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey" + url = ( + "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&" + "groupby=target&metric=sum__value&row_limit=5000&where=&having=&" + "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&" + "collapsed_fieldsets=&action={}&datasource_name=energy_usage&" + "datasource_id=1&datasource_type=table&previous_viz_type=sankey") db.session.commit() resp = self.client.get( @@ -147,7 +185,8 @@ def test_slices(self): for slc in db.session.query(Slc).all(): urls += [ (slc.slice_name, 'slice_url', slc.slice_url), - (slc.slice_name, 'slice_id_endpoint', '/caravel/slices/{}'.format(slc.id)), + (slc.slice_name, 'slice_id_endpoint', '/caravel/slices/{}'. + format(slc.id)), (slc.slice_name, 'json_endpoint', slc.viz.json_endpoint), (slc.slice_name, 'csv_endpoint', slc.viz.csv_endpoint), ] @@ -176,13 +215,20 @@ def test_misc(self): def test_shortner(self): self.login(username='admin') - data = "//caravel/explore/table/1/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey" + data = ( + "//caravel/explore/table/1/?viz_type=sankey&groupby=source&" + "groupby=target&metric=sum__value&row_limit=5000&where=&having=&" + "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=" + "Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=" + "energy_usage&datasource_id=1&datasource_type=table&" + "previous_viz_type=sankey") resp = self.client.post('/r/shortner/', data=data) assert '/r/' in resp.data.decode('utf-8') def test_save_dash(self, username='admin'): self.login(username=username) - dash = db.session.query(models.Dashboard).filter_by(slug="births").first() + dash = db.session.query(models.Dashboard).filter_by( + slug="births").first() positions = [] for i, slc in enumerate(dash.slices): d = { @@ -203,18 +249,24 @@ def test_save_dash(self, username='admin'): def test_add_slices(self, username='admin'): self.login(username=username) - dash = db.session.query(models.Dashboard).filter_by(slug="births").first() - new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first() - existing_slice = db.session.query(models.Slice).filter_by(slice_name="Name Cloud").first() + dash = db.session.query(models.Dashboard).filter_by( + slug="births").first() + new_slice = db.session.query(models.Slice).filter_by( + slice_name="Mapbox Long/Lat").first() + existing_slice = db.session.query(models.Slice).filter_by( + slice_name="Name Cloud").first() data = { - "slice_ids": [new_slice.data["slice_id"], existing_slice.data["slice_id"]] + "slice_ids": [new_slice.data["slice_id"], + existing_slice.data["slice_id"]] } url = '/caravel/add_slices/{}/'.format(dash.id) resp = self.client.post(url, data=dict(data=json.dumps(data))) assert "SLICES ADDED" in resp.data.decode('utf-8') - dash = db.session.query(models.Dashboard).filter_by(slug="births").first() - new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first() + dash = db.session.query(models.Dashboard).filter_by( + slug="births").first() + new_slice = db.session.query(models.Slice).filter_by( + slice_name="Mapbox Long/Lat").first() assert new_slice in dash.slices assert len(set(dash.slices)) == len(dash.slices) @@ -305,7 +357,6 @@ def test_public_user_dashboard_access(self): data = resp.data.decode('utf-8') assert "/caravel/dashboard/world_health/" not in data - def test_only_owners_can_save(self): dash = ( db.session @@ -337,26 +388,26 @@ def test_only_owners_can_save(self): SEGMENT_METADATA = [{ "id": "some_id", - "intervals": [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ], + "intervals": ["2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z"], "columns": { "__time": { "type": "LONG", "hasMultipleValues": False, - "size": 407240380, "cardinality": None, "errorMessage": None }, + "size": 407240380, "cardinality": None, "errorMessage": None}, "dim1": { "type": "STRING", "hasMultipleValues": False, - "size": 100000, "cardinality": 1944, "errorMessage": None }, + "size": 100000, "cardinality": 1944, "errorMessage": None}, "dim2": { "type": "STRING", "hasMultipleValues": True, - "size": 100000, "cardinality": 1504, "errorMessage": None }, + "size": 100000, "cardinality": 1504, "errorMessage": None}, "metric1": { "type": "FLOAT", "hasMultipleValues": False, - "size": 100000, "cardinality": None, "errorMessage": None } + "size": 100000, "cardinality": None, "errorMessage": None} }, "aggregators": { "metric1": { "type": "longSum", "name": "metric1", - "fieldName": "metric1" } + "fieldName": "metric1"} }, "size": 300000, "numRows": 5000000 @@ -422,7 +473,8 @@ def test_client(self, PyDruid): datasource_id = cluster.datasources[0].id db.session.commit() - resp = self.client.get('/caravel/explore/druid/{}/'.format(datasource_id)) + resp = self.client.get('/caravel/explore/druid/{}/'.format( + datasource_id)) assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8') nres = [ @@ -434,9 +486,15 @@ def test_client(self, PyDruid): instance.export_pandas.return_value = df instance.query_dict = {} instance.query_builder.last_query.query_dict = {} - resp = self.client.get('/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&include_search=false&metrics=count&groupby=name&flt_col_0=dim1&flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&action=&datasource_name=test_datasource&datasource_id={}&datasource_type=druid&previous_viz_type=table&json=true&force=true'.format(datasource_id, datasource_id)) + resp = self.client.get( + '/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&' + 'druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&' + 'include_search=false&metrics=count&groupby=name&flt_col_0=dim1&' + 'flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&' + 'action=&datasource_name=test_datasource&datasource_id={}&' + 'datasource_type=druid&previous_viz_type=table&json=true&' + 'force=true'.format(datasource_id, datasource_id)) assert "Canada" in resp.data.decode('utf-8') - if __name__ == '__main__': unittest.main() From 419c8678ea2ffc0b13c0a8d9c20bc5f04ca8169c Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Mon, 25 Jul 2016 11:47:14 -0700 Subject: [PATCH 07/14] Add query model --- caravel/models.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/caravel/models.py b/caravel/models.py index 5c46293d6b809..ad3e68927f3e3 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -4,6 +4,7 @@ from __future__ import print_function from __future__ import unicode_literals +import enum import functools import json import logging @@ -38,6 +39,7 @@ from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship from sqlalchemy.sql import table, literal_column, text, column +from sqlalchemy.types import Enum from sqlalchemy_utils import EncryptedType import caravel @@ -1701,3 +1703,59 @@ class FavStar(Model): class_name = Column(String(50)) obj_id = Column(Integer) dttm = Column(DateTime, default=func.now()) + + +class QueryResult(Model): + """ORM model for SQL query results""" + + __tablename__ = 'query_result' + id = Column(Integer, primary_key=True) + query_id = Column(Integer, ForeignKey('query.id'), nullable=False) + tmp_table_id = Column(Integer, ForeignKey('tables.id'), nullable=True) + expiration_date = Column(DateTime, default=func.now()) + # TODO(b.kyryliuk): add data preview, first 10000 columns or so. + + +class QueryStatus(Enum): + SCHEDULED = "SCHEDULED" + CANCELLED = "CANCELLED" + IN_PROGRESS = "IN_PROGRESS" + FINISHED = "FINISHED" + TIMED_OUT = "TIMED_OUT" + FAILED = "FAILED" + + +class Query(Model): + + """ORM model for SQL query""" + + __tablename__ = 'query' + id = Column(Integer, primary_key=True) + + database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) + table_ids = Column(Integer, ForeignKey('tables.id'), nullable=True) + + user_id = Column(Integer, ForeignKey('ab_user.id'), nullable=True) + result_id = Column( + Integer, ForeignKey('query_result.id'), nullable=True) + + name = Column(String(256)) + query = Column(String(10000)) + # Configured in the caravel config. + query_limit = Column(Integer) + # query_status = Column(Enum(QueryStatus)) + query_status = Column(Enum( + "SCHEDULED", + "CANCELLED", + "IN_PROGRESS", + "FINISHED", + "TIMED_OUT", + "FAILED", + name="query_status")) + + # 1..100 + query_progress = Column(Integer) + start_time = Column(DateTime, default=func.now()) + end_time = Column(DateTime) + # Time the query was accessed. + viewed_on = Column(DateTime, default=func.now()) From c778923ce5f6125596dd4f9898e33ec97d6a0108 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Mon, 25 Jul 2016 15:36:24 -0700 Subject: [PATCH 08/14] Remove QueryResult. Query has a tmp_table_name field that has all the data. --- .../versions/ad82a75afd82_add_query_model.py | 38 ++++++ caravel/models.py | 63 ++++------ caravel/tasks.py | 92 +++++++++++---- run_tests.sh | 1 + tests/caravel_test_config.py | 9 +- tests/celery_tests.py | 111 ++++++++++++++---- tests/core_tests.py | 26 +--- 7 files changed, 225 insertions(+), 115 deletions(-) create mode 100644 caravel/migrations/versions/ad82a75afd82_add_query_model.py diff --git a/caravel/migrations/versions/ad82a75afd82_add_query_model.py b/caravel/migrations/versions/ad82a75afd82_add_query_model.py new file mode 100644 index 0000000000000..8c5299cee1dde --- /dev/null +++ b/caravel/migrations/versions/ad82a75afd82_add_query_model.py @@ -0,0 +1,38 @@ +"""empty message + +Revision ID: ad82a75afd82 +Revises: f162a1dea4c4 +Create Date: 2016-07-25 17:48:12.771103 + +""" + +# revision identifiers, used by Alembic. +revision = 'ad82a75afd82' +down_revision = 'f162a1dea4c4' + +from alembic import op +import sqlalchemy as sa + +def upgrade(): + op.create_table('query', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('database_id', sa.Integer(), nullable=False), + sa.Column('table_names', sa.Integer(), nullable=True), + sa.Column('tmp_table_name', sa.String(length=64), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('query_status', sa.String(length=16), nullable=True), + sa.Column('query_name', sa.String(length=64), nullable=True), + sa.Column('query_text', sa.String(length=10000), nullable=True), + sa.Column('query_limit', sa.Integer(), nullable=True), + sa.Column('query_progress', sa.Integer(), nullable=True), + sa.Column('start_time', sa.BigInteger(), nullable=True), + sa.Column('end_time', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['database_id'], [u'dbs.id'], ), + sa.ForeignKeyConstraint(['table_names'], [u'tables.id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + +def downgrade(): + op.drop_table('query') diff --git a/caravel/models.py b/caravel/models.py index ad3e68927f3e3..88aa6385b05c4 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -4,7 +4,6 @@ from __future__ import print_function from __future__ import unicode_literals -import enum import functools import json import logging @@ -33,13 +32,12 @@ from pydruid.utils.having import Aggregation from six import string_types from sqlalchemy import ( - Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date, - Table, create_engine, MetaData, desc, asc, select, and_, func) + BigInteger, Column, Integer, String, ForeignKey, Text, Boolean, DateTime, + Date, Table, create_engine, MetaData, desc, asc, select, and_, func) from sqlalchemy.engine import reflection from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship from sqlalchemy.sql import table, literal_column, text, column -from sqlalchemy.types import Enum from sqlalchemy_utils import EncryptedType import caravel @@ -1705,24 +1703,13 @@ class FavStar(Model): dttm = Column(DateTime, default=func.now()) -class QueryResult(Model): - """ORM model for SQL query results""" - - __tablename__ = 'query_result' - id = Column(Integer, primary_key=True) - query_id = Column(Integer, ForeignKey('query.id'), nullable=False) - tmp_table_id = Column(Integer, ForeignKey('tables.id'), nullable=True) - expiration_date = Column(DateTime, default=func.now()) - # TODO(b.kyryliuk): add data preview, first 10000 columns or so. - - -class QueryStatus(Enum): - SCHEDULED = "SCHEDULED" - CANCELLED = "CANCELLED" - IN_PROGRESS = "IN_PROGRESS" - FINISHED = "FINISHED" - TIMED_OUT = "TIMED_OUT" - FAILED = "FAILED" +class QueryStatus: + SCHEDULED = 'SCHEDULED' + CANCELLED = 'CANCELLED' + IN_PROGRESS = 'IN_PROGRESS' + FINISHED = 'FINISHED' + TIMED_OUT = 'TIMED_OUT' + FAILED = 'FAILED' class Query(Model): @@ -1733,29 +1720,23 @@ class Query(Model): id = Column(Integer, primary_key=True) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) - table_ids = Column(Integer, ForeignKey('tables.id'), nullable=True) + table_names = Column(Integer, ForeignKey('tables.id'), nullable=True) + # Add comma seperate table names as not all of them would be in the + # tables table. + # Store the tmp table into the DB only if the user asks for it. + tmp_table_name = Column(String(64)) user_id = Column(Integer, ForeignKey('ab_user.id'), nullable=True) - result_id = Column( - Integer, ForeignKey('query_result.id'), nullable=True) - name = Column(String(256)) - query = Column(String(10000)) - # Configured in the caravel config. + # models.QueryStatus + query_status = Column(String(16)) + + query_name = Column(String(64)) + query_text = Column(String(10000)) + # Could be configured in the caravel config query_limit = Column(Integer) - # query_status = Column(Enum(QueryStatus)) - query_status = Column(Enum( - "SCHEDULED", - "CANCELLED", - "IN_PROGRESS", - "FINISHED", - "TIMED_OUT", - "FAILED", - name="query_status")) # 1..100 query_progress = Column(Integer) - start_time = Column(DateTime, default=func.now()) - end_time = Column(DateTime) - # Time the query was accessed. - viewed_on = Column(DateTime, default=func.now()) + start_time = Column(BigInteger) + end_time = Column(BigInteger) diff --git a/caravel/tasks.py b/caravel/tasks.py index 0ea5ab0569a1e..f45f82b6e18a8 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -4,20 +4,66 @@ from sqlalchemy import select, text from sqlalchemy.sql.expression import TextAsFrom import pandas as pd +import time celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG')) +def init_query(database_id, sql, user_id): + milis = int(time.time() * 1000) + + query = models.Query() + query.user_id = user_id + query.database_id = database_id + if app.config.get('SQL_MAX_ROW'): + query.query_limit = app.config.get('SQL_MAX_ROW') + query.query_name = str(milis) + query.query_text = sql + # TODO(bkyryliuk): run explain query to derive the tables and fill in the + # table_ids + # TODO(bkyryliuk): check the user permissions + query.start_time = milis + query.query_status = models.QueryStatus.IN_PROGRESS + return query + + +# def create_table_as(sql, table_name): +# return "CREATE TABLE %s AS %s".format(sql, table_name) + + @celery_app.task -def get_sql_results(database_id, sql): - session = db.session() - mydb = session.query(models.Database).filter_by(id=database_id).first() - return get_sql_results_internal(sql, session, mydb) +def get_sql_results(database_id, sql, user_id): + query_session = db.session() + db_to_query = query_session.query(models.Database).filter_by( + id=database_id).first() + if not db_to_query: + return json.dumps( + {'msg': "Database with id {0} is missing.".format(database_id)}) + query = init_query(database_id, sql, user_id) + query_session.add(query) + query_session.flush() -# TODO(b.kyryliuk): merge the changes made in the carapal first + query_result, success = get_sql_results_internal(sql, db_to_query) + query_session.commit() + # TODO: update the query values + + query.end_time = int(time.time() * 1000) + if success: + query.query_status = models.QueryStatus.FINISHED + # TODO(bkyryliuk): fill in query.tmp_table_name + else: + query.query_status = models.QueryStatus.FAILED + + query_session.commit() + query_session.close() + # TODO(bkyryliuk): return the tmp table / query_id + return query_result + + +# TODO(bkyryliuk): merge the changes made in the carapal first # before merging this PR. -def get_sql_results_internal(sql, session, mydb): +def get_sql_results_internal(sql, db_to_query): """Get the SQL query resulst from the give session and db connection. Attributes: @@ -28,9 +74,8 @@ def get_sql_results_internal(sql, session, mydb): Returns: string: table in the html format. """ - content = "" - eng = mydb.get_sqla_engine() - if mydb: + try: + eng = db_to_query.get_sqla_engine() if app.config.get('SQL_MAX_ROW'): sql = sql.strip().strip(';') qry = ( @@ -38,17 +83,18 @@ def get_sql_results_internal(sql, session, mydb): .select_from(TextAsFrom(text(sql), ['*']).alias('inner_qry')) .limit(app.config.get('SQL_MAX_ROW')) ) - sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) - try: - df = pd.read_sql_query(sql=sql, con=eng) - # TODO(b.kyryliuk): refactore the output to be json instead of html - data = { - 'columns': [c for c in df.columns], - 'data': df.to_dict(orient='records'), - } - return json.dumps(data, allow_nan=False) - except Exception as e: - content = json.dumps({'msg': e.message}) - session.commit() - - return content + # if db_to_query.select_as_create_table_as: + # sql = sql.strip().strip(';') + # # TODO(bkyryliuk): figure out if the query is select query. + # sql = create_table_as(sql, int(round(time.time() * 1000))) + + sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) + df = pd.read_sql_query(sql=sql, con=eng) + # TODO(bkyryliuk): refactore the output to be json instead of html + data = { + 'columns': [c for c in df.columns], + 'data': df.to_dict(orient='records'), + } + return json.dumps(data, allow_nan=False), True + except Exception as e: + return json.dumps({'msg': e.message}), False diff --git a/run_tests.sh b/run_tests.sh index 95de796a98748..faf19251a9ba4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,3 +9,4 @@ set -e caravel/bin/caravel db upgrade caravel/bin/caravel version -v python setup.py nosetests + diff --git a/tests/caravel_test_config.py b/tests/caravel_test_config.py index e5d77991a186b..203a5f5025d33 100644 --- a/tests/caravel_test_config.py +++ b/tests/caravel_test_config.py @@ -10,11 +10,14 @@ if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ: SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI') +SQL_CELERY_DB_FILE_PATH = '/tmp/celerydb.sqlite' +SQL_CELERY_RESULTS_DB_FILE_PATH = '/tmp/celery_results.sqlite' + class CeleryConfig(object): - BROKER_URL = 'sqla+sqlite:////tmp/celerydb.sqlite' + BROKER_URL = 'sqla+sqlite:///' + SQL_CELERY_DB_FILE_PATH CELERY_IMPORTS = ('caravel.tasks', ) - CELERY_RESULT_BACKEND = 'db+sqlite:////tmp/celery_results.sqlite' - CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '1/s'}} + CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} CONCURRENCY = 1 CELERY_CONFIG = CeleryConfig diff --git a/tests/celery_tests.py b/tests/celery_tests.py index 187a3074949b9..ec1996add6b54 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -11,7 +11,7 @@ import time import unittest import caravel -from caravel import app, utils, appbuilder, tasks +from caravel import app, appbuilder, db, models, tasks, utils class CeleryConfig(object): @@ -40,6 +40,15 @@ def __init__(self, *args, **kwargs): @classmethod def setUpClass(cls): + try: + os.remove(app.config.get('SQL_CELERY_DB_FILE_PATH')) + except OSError: + pass + try: + os.remove(app.config.get('SQL_CELERY_RESULTS_DB_FILE_PATH')) + except OSError: + pass + worker_command = BASE_DIR + "/bin/caravel worker" subprocess.Popen( worker_command, shell=True, stdout=subprocess.PIPE) @@ -48,6 +57,11 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + subprocess.call( + "ps auxww | grep 'celeryd' | awk '{print $2}' | " + "xargs kill -9", + shell=True + ) subprocess.call( "ps auxww | grep 'caravel worker' | awk '{print $2}' | " "xargs kill -9", @@ -62,53 +76,104 @@ def tearDown(self): def test_run_async_query_delay_get(self): # DB #0 doesn't exist. - celery_task = tasks.get_sql_results.delay(0, "SELECT * FROM dontexist") - self.assertRaises(AttributeError, celery_task.get) - result1 = tasks.get_sql_results.delay( - 1, "SELECT * FROM dontexist").get() + 0, "SELECT * FROM dontexist", 1).get() expected_result1 = ( - '{"msg": "(sqlite3.OperationalError) no such table: dontexist"}') + '{"msg": "Database with id 0 is missing."}') self.assertEqual(expected_result1, result1) + session1 = db.create_scoped_session() + query1 = session1.query(models.Query).filter_by( + query_text="SELECT * FROM dontexist").first() + session1.close() + self.assertIsNone(query1) + + session2 = db.create_scoped_session() + query2 = session2.query(models.Query).filter_by( + query_text="SELECT * FROM dontexist1").first() + self.assertEqual(models.QueryStatus.FAILED, query2.query_status) + session2.close() result2 = tasks.get_sql_results.delay( - 1, "SELECT * FROM ab_permission WHERE name='can_select_star'").get() + 1, "SELECT * FROM dontexist1", 1).get() + print("blabla") + print(result2) expected_result2 = ( - '{"data": [{"id": 20, "name": "can_select_star"}], "columns":' - ' ["id", "name"]}') + '{"msg": "(sqlite3.OperationalError) no such table: dontexist1"}') self.assertEqual(expected_result2, result2) + session2 = db.create_scoped_session() + query2 = session2.query(models.Query).filter_by( + query_text="SELECT * FROM dontexist1").first() + self.assertEqual(models.QueryStatus.FAILED, query2.query_status) + session2.close() result3 = tasks.get_sql_results.delay( - 1, "SELECT * FROM ab_permission WHERE id=666").get() - expected_result3 = '{"data": [], "columns": ["id", "name"]}' + 1, "SELECT name FROM ab_permission WHERE name='can_select_star'", + 1).get() + expected_result3 = ( + '{"data": [{"name": "can_select_star"}], "columns": ["name"]}') self.assertEqual(expected_result3, result3) + session3 = db.create_scoped_session() + query3 = session3.query(models.Query).filter_by( + query_text= + "SELECT name FROM ab_permission WHERE name='can_select_star'").first() + self.assertEqual(models.QueryStatus.FINISHED, query3.query_status) + session3.close() + + result4 = tasks.get_sql_results.delay( + 1, "SELECT * FROM ab_permission WHERE id=666", 1).get() + expected_result4 = '{"data": [], "columns": ["id", "name"]}' + self.assertEqual(expected_result4, result4) + session4 = db.create_scoped_session() + query4 = session4.query(models.Query).filter_by( + query_text="SELECT * FROM ab_permission WHERE id=666").first() + self.assertEqual(models.QueryStatus.FINISHED, query4.query_status) + session4.close() def test_run_async_query_delay(self): - celery_task1 = tasks.get_sql_results.delay(0, "SELECT * FROM dontexist") + celery_task1 = tasks.get_sql_results.delay( + 0, "SELECT * FROM dontexist", 1) celery_task2 = tasks.get_sql_results.delay( - 1, "SELECT * FROM dontexist") + 1, "SELECT * FROM dontexist1", 1) celery_task3 = tasks.get_sql_results.delay( - 1, "SELECT * FROM ab_permission WHERE name='can_select_star'") + 1, "SELECT name FROM ab_permission WHERE name='can_select_star'", 1) celery_task4 = tasks.get_sql_results.delay( - 1, "SELECT * FROM ab_permission WHERE id=666") + 1, "SELECT * FROM ab_permission WHERE id=666", 1) - time.sleep(1) + time.sleep(2) # DB #0 doesn't exist. - self.assertRaises(AttributeError, celery_task1.get) - + expected_result1 = ( + '{"msg": "Database with id 0 is missing."}') + self.assertEqual(expected_result1, celery_task1.get()) + session2 = db.create_scoped_session() + query2 = session2.query(models.Query).filter_by( + query_text="SELECT * FROM dontexist1").first() + self.assertEqual(models.QueryStatus.FAILED, query2.query_status) expected_result2 = ( - '{"msg": "(sqlite3.OperationalError) no such table: dontexist"}') + '{"msg": "(sqlite3.OperationalError) no such table: dontexist1"}') self.assertEqual(expected_result2, celery_task2.get()) - expected_result3 = ( - '{"data": [{"id": 20, "name": "can_select_star"}], "columns":' - ' ["id", "name"]}') + '{"data": [{"name": "can_select_star"}], "columns": ["name"]}') self.assertEqual(expected_result3, celery_task3.get()) - expected_result4 = '{"data": [], "columns": ["id", "name"]}' self.assertEqual(expected_result4, celery_task4.get()) + session = db.create_scoped_session() + query1 = session.query(models.Query).filter_by( + query_text="SELECT * FROM dontexist").first() + self.assertIsNone(query1) + query2 = session.query(models.Query).filter_by( + query_text="SELECT * FROM dontexist1").first() + self.assertEqual(models.QueryStatus.FAILED, query2.query_status) + query3 = session.query(models.Query).filter_by( + query_text= + "SELECT name FROM ab_permission WHERE name='can_select_star'").first() + self.assertEqual(models.QueryStatus.FINISHED, query3.query_status) + query4 = session.query(models.Query).filter_by( + query_text="SELECT * FROM ab_permission WHERE id=666").first() + self.assertEqual(models.QueryStatus.FINISHED, query4.query_status) + session.close() + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/core_tests.py b/tests/core_tests.py index 2a215ffb2c088..3d93999a079b7 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -8,9 +8,7 @@ import doctest import json import imp -import mock import os -import sqlalchemy import unittest from mock import Mock, patch @@ -20,7 +18,7 @@ import caravel from caravel import app, db, models, utils, appbuilder, tasks -from caravel.models import DruidCluster, DruidDatasource +from caravel.models import DruidCluster os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' @@ -127,28 +125,6 @@ def setUp(self): def tearDown(self): pass - @mock.patch('caravel.db') - @mock.patch('pandas.read_sql_query') - def test_celery_sql_execution( - self, caravel_db_mock, unused_pd_read_sql_query_mock): - mydb = mock.MagicMock() - sqlite_connection = sqlalchemy.create_engine( - "sqlite:///tmp/caravel_unittests.db") - mydb.get_sqla_engine = mock.MagicMock(return_value=sqlite_connection) - tasks.get_sql_results_internal( - "SELECT * FROM TABLE", mock.MagicMock(), mydb) - caravel_db_mock.assert_called_once_with( - con=sqlite_connection, - sql="SELECT * \nFROM (SELECT * FROM TABLE) AS inner_qry\n" - " LIMIT 1000 OFFSET 0") - - caravel_db_mock.reset_mock() - app.config['SQL_MAX_ROW'] = None - tasks.get_sql_results_internal( - "SELECT * FROM TABLE", mock.MagicMock(), mydb) - caravel_db_mock.assert_called_once_with( - con=sqlite_connection, sql="SELECT * FROM TABLE") - def test_save_slice(self): self.login(username='admin') From 6554d1e56af89a2dcd8a7139b9608b3ee01cc560 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Wed, 27 Jul 2016 22:39:51 -0700 Subject: [PATCH 09/14] Add create table as wrapper. --- caravel/migrations/versions/df564881a5eb_.py | 21 ++++ caravel/models.py | 1 + caravel/tasks.py | 80 ++++++++++---- tests/celery_tests.py | 110 +++++++++---------- tests/core_tests.py | 8 -- 5 files changed, 129 insertions(+), 91 deletions(-) create mode 100644 caravel/migrations/versions/df564881a5eb_.py diff --git a/caravel/migrations/versions/df564881a5eb_.py b/caravel/migrations/versions/df564881a5eb_.py new file mode 100644 index 0000000000000..027fd0e124fa4 --- /dev/null +++ b/caravel/migrations/versions/df564881a5eb_.py @@ -0,0 +1,21 @@ +"""empty message + +Revision ID: df564881a5eb +Revises: ad82a75afd82 +Create Date: 2016-07-27 17:35:40.953223 + +""" + +# revision identifiers, used by Alembic. +revision = 'df564881a5eb' +down_revision = 'ad82a75afd82' + +from alembic import op +import sqlalchemy as sa + +def upgrade(): + op.add_column('dbs', sa.Column('select_as_create_table_as', sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column('dbs', 'select_as_create_table_as') diff --git a/caravel/models.py b/caravel/models.py index 88aa6385b05c4..c5a11e1649b20 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -378,6 +378,7 @@ class Database(Model, AuditMixinNullable): sqlalchemy_uri = Column(String(1024)) password = Column(EncryptedType(String(1024), config.get('SECRET_KEY'))) cache_timeout = Column(Integer) + select_as_create_table_as = Column(Boolean, default=False) extra = Column(Text, default=textwrap.dedent("""\ { "metadata_params": {}, diff --git a/caravel/tasks.py b/caravel/tasks.py index f45f82b6e18a8..e56a7a9ad3f81 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -1,7 +1,8 @@ import celery from caravel import db, models, app import json -from sqlalchemy import select, text +from sqlalchemy import create_engine, select, text +from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.sql.expression import TextAsFrom import pandas as pd import time @@ -10,6 +11,13 @@ def init_query(database_id, sql, user_id): + """Initializes the models.Query object. + + :param database_id: integer + :param sql: sql query that will be executed + :param user_id: interger + :return: models.Query + """ milis = int(time.time() * 1000) query = models.Query() @@ -20,33 +28,57 @@ def init_query(database_id, sql, user_id): query.query_name = str(milis) query.query_text = sql # TODO(bkyryliuk): run explain query to derive the tables and fill in the - # table_ids + # table_ids # TODO(bkyryliuk): check the user permissions query.start_time = milis query.query_status = models.QueryStatus.IN_PROGRESS return query -# def create_table_as(sql, table_name): -# return "CREATE TABLE %s AS %s".format(sql, table_name) +def create_table_as(sql, table_name): + """Reformats the query into the create table as query. + + :param sql: string, sql query that will be executed + :param table_name: string, will contain the results of the query execution + :return: string, create table as query + """ + return "CREATE TABLE %s AS %s" % (table_name, sql) +def get_session(): + """Creates new SQLAlchemy scoped_session.""" + engine = create_engine( + app.config.get('SQLALCHEMY_DATABASE_URI'), convert_unicode=True) + return scoped_session(sessionmaker( + autocommit=False, autoflush=False, bind=engine)) + @celery_app.task def get_sql_results(database_id, sql, user_id): - query_session = db.session() - db_to_query = query_session.query(models.Database).filter_by( - id=database_id).first() + """Executes the sql query returns the results. + + :param database_id: integer + :param sql: string, query that will be executed + :param user_id: integer + :return: dataframe, query result + """ + # Create a separate session, reusing the db.session leads to the + # concurrency issues. + query_session = get_session() + try: + db_to_query = query_session.query(models.Database).filter_by( + id=database_id).first() + except Exception as e: + return json.dumps({'msg': str(e)}) + if not db_to_query: return json.dumps( {'msg': "Database with id {0} is missing.".format(database_id)}) query = init_query(database_id, sql, user_id) query_session.add(query) - query_session.flush() + query_session.commit() query_result, success = get_sql_results_internal(sql, db_to_query) - query_session.commit() - # TODO: update the query values query.end_time = int(time.time() * 1000) if success: @@ -56,7 +88,6 @@ def get_sql_results(database_id, sql, user_id): query.query_status = models.QueryStatus.FAILED query_session.commit() - query_session.close() # TODO(bkyryliuk): return the tmp table / query_id return query_result @@ -66,13 +97,9 @@ def get_sql_results(database_id, sql, user_id): def get_sql_results_internal(sql, db_to_query): """Get the SQL query resulst from the give session and db connection. - Attributes: - sql (string): SQL query that will be executed - session (caravel.db.session()): DB session - mydb (sqlalchemy.orm.query.Query): source of all SELECT statements - generated by the ORM - Returns: - string: table in the html format. + :param sql: string, query that will be executed + :param db_to_query: models.Database to query + :return: (dataframe, boolean), results and the status """ try: eng = db_to_query.get_sqla_engine() @@ -83,12 +110,13 @@ def get_sql_results_internal(sql, db_to_query): .select_from(TextAsFrom(text(sql), ['*']).alias('inner_qry')) .limit(app.config.get('SQL_MAX_ROW')) ) - # if db_to_query.select_as_create_table_as: - # sql = sql.strip().strip(';') - # # TODO(bkyryliuk): figure out if the query is select query. - # sql = create_table_as(sql, int(round(time.time() * 1000))) + sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) + if db_to_query.select_as_create_table_as: + sql = sql.strip().strip(';') + # TODO(bkyryliuk): figure out if the query is select query. + sql = create_table_as( + sql, "query_" + str(int(round(time.time() * 1000)))) - sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) df = pd.read_sql_query(sql=sql, con=eng) # TODO(bkyryliuk): refactore the output to be json instead of html data = { @@ -97,4 +125,8 @@ def get_sql_results_internal(sql, db_to_query): } return json.dumps(data, allow_nan=False), True except Exception as e: - return json.dumps({'msg': e.message}), False + if hasattr(e, 'message'): + return json.dumps({'msg': e.message}), False + else: + return json.dumps({'msg': str(e)}), False + diff --git a/tests/celery_tests.py b/tests/celery_tests.py index ec1996add6b54..72c5e6154f8a3 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -1,15 +1,13 @@ -"""Unit tests for Caravel Celery worker""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +'''Unit tests for Caravel Celery worker''' import imp +import json import subprocess import os -os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' - import time import unittest + +# Set environ before caravel module is imported. +os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' import caravel from caravel import app, appbuilder, db, models, tasks, utils @@ -21,8 +19,8 @@ class CeleryConfig(object): CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} app.config['CELERY_CONFIG'] = CeleryConfig -BASE_DIR = app.config.get("BASE_DIR") -cli = imp.load_source('cli', BASE_DIR + "/bin/caravel") +BASE_DIR = app.config.get('BASE_DIR') +cli = imp.load_source('cli', BASE_DIR + '/bin/caravel') class CeleryTestCase(unittest.TestCase): @@ -49,12 +47,11 @@ def setUpClass(cls): except OSError: pass - worker_command = BASE_DIR + "/bin/caravel worker" + worker_command = BASE_DIR + '/bin/caravel worker' subprocess.Popen( worker_command, shell=True, stdout=subprocess.PIPE) cli.load_examples(load_test_data=True) - @classmethod def tearDownClass(cls): subprocess.call( @@ -77,103 +74,98 @@ def tearDown(self): def test_run_async_query_delay_get(self): # DB #0 doesn't exist. result1 = tasks.get_sql_results.delay( - 0, "SELECT * FROM dontexist", 1).get() - expected_result1 = ( - '{"msg": "Database with id 0 is missing."}') - self.assertEqual(expected_result1, result1) + 0, 'SELECT * FROM dontexist', 1).get() + expected_result1 = {'msg': 'Database with id 0 is missing.'} + self.assertEqual(expected_result1, json.loads(result1)) session1 = db.create_scoped_session() query1 = session1.query(models.Query).filter_by( - query_text="SELECT * FROM dontexist").first() + query_text='SELECT * FROM dontexist').first() session1.close() self.assertIsNone(query1) session2 = db.create_scoped_session() query2 = session2.query(models.Query).filter_by( - query_text="SELECT * FROM dontexist1").first() + query_text='SELECT * FROM dontexist1').first() self.assertEqual(models.QueryStatus.FAILED, query2.query_status) session2.close() result2 = tasks.get_sql_results.delay( - 1, "SELECT * FROM dontexist1", 1).get() - print("blabla") - print(result2) - expected_result2 = ( - '{"msg": "(sqlite3.OperationalError) no such table: dontexist1"}') - self.assertEqual(expected_result2, result2) + 1, 'SELECT * FROM dontexist1', 1).get() + self.assertTrue('msg' in result2) session2 = db.create_scoped_session() query2 = session2.query(models.Query).filter_by( - query_text="SELECT * FROM dontexist1").first() + query_text='SELECT * FROM dontexist1').first() self.assertEqual(models.QueryStatus.FAILED, query2.query_status) session2.close() - result3 = tasks.get_sql_results.delay( - 1, "SELECT name FROM ab_permission WHERE name='can_select_star'", - 1).get() - expected_result3 = ( - '{"data": [{"name": "can_select_star"}], "columns": ["name"]}') - self.assertEqual(expected_result3, result3) + where_query = ( + "SELECT name FROM ab_permission WHERE name='can_select_star'") + result3 = tasks.get_sql_results.delay(1, where_query, 1).get() + expected_result3 = { + 'columns': ['name'], 'data': [{'name': 'can_select_star'}]} + self.assertEqual( + sorted(expected_result3.items()), + sorted(json.loads(result3).items())) session3 = db.create_scoped_session() query3 = session3.query(models.Query).filter_by( - query_text= - "SELECT name FROM ab_permission WHERE name='can_select_star'").first() + query_text=where_query).first() self.assertEqual(models.QueryStatus.FINISHED, query3.query_status) session3.close() result4 = tasks.get_sql_results.delay( - 1, "SELECT * FROM ab_permission WHERE id=666", 1).get() - expected_result4 = '{"data": [], "columns": ["id", "name"]}' - self.assertEqual(expected_result4, result4) + 1, 'SELECT * FROM ab_permission WHERE id=666', 1).get() + expected_result4 = {'columns': ['id', 'name'], 'data': []} + self.assertEqual(expected_result4, json.loads(result4)) session4 = db.create_scoped_session() query4 = session4.query(models.Query).filter_by( - query_text="SELECT * FROM ab_permission WHERE id=666").first() + query_text='SELECT * FROM ab_permission WHERE id=666').first() self.assertEqual(models.QueryStatus.FINISHED, query4.query_status) session4.close() def test_run_async_query_delay(self): celery_task1 = tasks.get_sql_results.delay( - 0, "SELECT * FROM dontexist", 1) + 0, 'SELECT * FROM dontexist', 1) celery_task2 = tasks.get_sql_results.delay( - 1, "SELECT * FROM dontexist1", 1) - celery_task3 = tasks.get_sql_results.delay( - 1, "SELECT name FROM ab_permission WHERE name='can_select_star'", 1) + 1, 'SELECT * FROM dontexist1', 1) + where_query = ( + "SELECT name FROM ab_permission WHERE name='can_select_star'") + celery_task3 = tasks.get_sql_results.delay(1, where_query, 1) celery_task4 = tasks.get_sql_results.delay( - 1, "SELECT * FROM ab_permission WHERE id=666", 1) + 1, 'SELECT * FROM ab_permission WHERE id=666', 1) time.sleep(2) # DB #0 doesn't exist. - expected_result1 = ( - '{"msg": "Database with id 0 is missing."}') - self.assertEqual(expected_result1, celery_task1.get()) + expected_result1 = {'msg': 'Database with id 0 is missing.'} + self.assertEqual(expected_result1, json.loads(celery_task1.get())) session2 = db.create_scoped_session() query2 = session2.query(models.Query).filter_by( - query_text="SELECT * FROM dontexist1").first() + query_text='SELECT * FROM dontexist1').first() self.assertEqual(models.QueryStatus.FAILED, query2.query_status) - expected_result2 = ( - '{"msg": "(sqlite3.OperationalError) no such table: dontexist1"}') - self.assertEqual(expected_result2, celery_task2.get()) - expected_result3 = ( - '{"data": [{"name": "can_select_star"}], "columns": ["name"]}') - self.assertEqual(expected_result3, celery_task3.get()) - expected_result4 = '{"data": [], "columns": ["id", "name"]}' - self.assertEqual(expected_result4, celery_task4.get()) + self.assertTrue('msg' in celery_task2.get()) + expected_result3 = { + 'columns': ['name'], 'data': [{'name': 'can_select_star'}]} + self.assertEqual( + sorted(expected_result3.items()), + sorted(json.loads(celery_task3.get()).items())) + expected_result4 = {'columns': ['id', 'name'], 'data': []} + self.assertEqual(expected_result4, json.loads(celery_task4.get())) session = db.create_scoped_session() query1 = session.query(models.Query).filter_by( - query_text="SELECT * FROM dontexist").first() + query_text='SELECT * FROM dontexist').first() self.assertIsNone(query1) query2 = session.query(models.Query).filter_by( - query_text="SELECT * FROM dontexist1").first() + query_text='SELECT * FROM dontexist1').first() self.assertEqual(models.QueryStatus.FAILED, query2.query_status) query3 = session.query(models.Query).filter_by( - query_text= - "SELECT name FROM ab_permission WHERE name='can_select_star'").first() + query_text=where_query).first() self.assertEqual(models.QueryStatus.FINISHED, query3.query_status) query4 = session.query(models.Query).filter_by( - query_text="SELECT * FROM ab_permission WHERE id=666").first() + query_text='SELECT * FROM ab_permission WHERE id=666').first() self.assertEqual(models.QueryStatus.FINISHED, query4.query_status) session.close() if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/core_tests.py b/tests/core_tests.py index 3d93999a079b7..77212094859cb 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -27,14 +27,6 @@ app.config['SECRET_KEY'] = 'thisismyscretkey' app.config['WTF_CSRF_ENABLED'] = False app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True - -class CeleryConfig(object): - BROKER_URL = 'sqla+sqlite:////tmp/celerydb.sqlite' - CELERY_IMPORTS = ('caravel.tasks',) - CELERY_RESULT_BACKEND = 'db+sqlite:////tmp/celery_results.sqlite' - CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} -app.config['CELERY_CONFIG'] = CeleryConfig - BASE_DIR = app.config.get("BASE_DIR") cli = imp.load_source('cli', BASE_DIR + "/bin/caravel") From 14bbe762fef17f4fbc2bacd1d294651e1b1726a5 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Mon, 1 Aug 2016 10:32:01 -0700 Subject: [PATCH 10/14] Create table as --- caravel/bin/caravel | 3 - .../versions/ad82a75afd82_add_query_model.py | 21 +- caravel/migrations/versions/df564881a5eb_.py | 21 -- caravel/models.py | 24 +- caravel/tasks.py | 225 ++++++++---- caravel/views.py | 54 +-- tests/caravel_test_config.py | 7 +- tests/celery_tests.py | 329 +++++++++++++++--- tests/core_tests.py | 22 +- 9 files changed, 505 insertions(+), 201 deletions(-) delete mode 100644 caravel/migrations/versions/df564881a5eb_.py diff --git a/caravel/bin/caravel b/caravel/bin/caravel index 5db60e823e1dc..51d0f63c4e1fa 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -147,8 +147,5 @@ def worker(): c_worker.run(**options) -if config.get('CELERY_CONFIG'): - print(config.get('CELERY_CONFIG').BROKER_URL) - if __name__ == "__main__": manager.run() diff --git a/caravel/migrations/versions/ad82a75afd82_add_query_model.py b/caravel/migrations/versions/ad82a75afd82_add_query_model.py index 8c5299cee1dde..4794f416de07f 100644 --- a/caravel/migrations/versions/ad82a75afd82_add_query_model.py +++ b/caravel/migrations/versions/ad82a75afd82_add_query_model.py @@ -1,4 +1,4 @@ -"""empty message +"""Update models to support storing the queries. Revision ID: ad82a75afd82 Revises: f162a1dea4c4 @@ -17,22 +17,23 @@ def upgrade(): op.create_table('query', sa.Column('id', sa.Integer(), nullable=False), sa.Column('database_id', sa.Integer(), nullable=False), - sa.Column('table_names', sa.Integer(), nullable=True), sa.Column('tmp_table_name', sa.String(length=64), nullable=True), sa.Column('user_id', sa.Integer(), nullable=True), - sa.Column('query_status', sa.String(length=16), nullable=True), - sa.Column('query_name', sa.String(length=64), nullable=True), - sa.Column('query_text', sa.String(length=10000), nullable=True), - sa.Column('query_limit', sa.Integer(), nullable=True), - sa.Column('query_progress', sa.Integer(), nullable=True), - sa.Column('start_time', sa.BigInteger(), nullable=True), - sa.Column('end_time', sa.BigInteger(), nullable=True), + sa.Column('status', sa.String(length=16), nullable=True), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('sql', sa.Text, nullable=True), + sa.Column('limit', sa.Integer(), nullable=True), + sa.Column('progress', sa.Integer(), nullable=True), + sa.Column('start_time', sa.DateTime(), nullable=True), + sa.Column('end_time', sa.DateTime(), nullable=True), sa.ForeignKeyConstraint(['database_id'], [u'dbs.id'], ), - sa.ForeignKeyConstraint(['table_names'], [u'tables.id'], ), sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ), sa.PrimaryKeyConstraint('id') ) + op.add_column('dbs', sa.Column('select_as_create_table_as', sa.Boolean(), + nullable=True)) def downgrade(): op.drop_table('query') + op.drop_column('dbs', 'select_as_create_table_as') diff --git a/caravel/migrations/versions/df564881a5eb_.py b/caravel/migrations/versions/df564881a5eb_.py deleted file mode 100644 index 027fd0e124fa4..0000000000000 --- a/caravel/migrations/versions/df564881a5eb_.py +++ /dev/null @@ -1,21 +0,0 @@ -"""empty message - -Revision ID: df564881a5eb -Revises: ad82a75afd82 -Create Date: 2016-07-27 17:35:40.953223 - -""" - -# revision identifiers, used by Alembic. -revision = 'df564881a5eb' -down_revision = 'ad82a75afd82' - -from alembic import op -import sqlalchemy as sa - -def upgrade(): - op.add_column('dbs', sa.Column('select_as_create_table_as', sa.Boolean(), nullable=True)) - - -def downgrade(): - op.drop_column('dbs', 'select_as_create_table_as') diff --git a/caravel/models.py b/caravel/models.py index c5a11e1649b20..6dbeb53c6a313 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -32,8 +32,9 @@ from pydruid.utils.having import Aggregation from six import string_types from sqlalchemy import ( - BigInteger, Column, Integer, String, ForeignKey, Text, Boolean, DateTime, - Date, Table, create_engine, MetaData, desc, asc, select, and_, func) + Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date, Table, + create_engine, MetaData, desc, asc, select, and_, func +) from sqlalchemy.engine import reflection from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import relationship @@ -378,7 +379,7 @@ class Database(Model, AuditMixinNullable): sqlalchemy_uri = Column(String(1024)) password = Column(EncryptedType(String(1024), config.get('SECRET_KEY'))) cache_timeout = Column(Integer) - select_as_create_table_as = Column(Boolean, default=False) + select_as_create_table_as = Column(Boolean, default=True) extra = Column(Text, default=textwrap.dedent("""\ { "metadata_params": {}, @@ -1721,23 +1722,20 @@ class Query(Model): id = Column(Integer, primary_key=True) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) - table_names = Column(Integer, ForeignKey('tables.id'), nullable=True) - # Add comma seperate table names as not all of them would be in the - # tables table. # Store the tmp table into the DB only if the user asks for it. tmp_table_name = Column(String(64)) user_id = Column(Integer, ForeignKey('ab_user.id'), nullable=True) # models.QueryStatus - query_status = Column(String(16)) + status = Column(String(16)) - query_name = Column(String(64)) - query_text = Column(String(10000)) + name = Column(String(64)) + sql = Column(Text) # Could be configured in the caravel config - query_limit = Column(Integer) + limit = Column(Integer) # 1..100 - query_progress = Column(Integer) - start_time = Column(BigInteger) - end_time = Column(BigInteger) + progress = Column(Integer) + start_time = Column(DateTime) + end_time = Column(DateTime) diff --git a/caravel/tasks.py b/caravel/tasks.py index e56a7a9ad3f81..62a3a09f563c9 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -1,48 +1,124 @@ import celery -from caravel import db, models, app -import json +from caravel import models, app, utils +from datetime import datetime +import logging from sqlalchemy import create_engine, select, text from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.sql.expression import TextAsFrom +import sqlparse import pandas as pd -import time celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG')) -def init_query(database_id, sql, user_id): +def init_query(database_id, sql, user_id, start_time, tmp_table_name): """Initializes the models.Query object. :param database_id: integer :param sql: sql query that will be executed - :param user_id: interger + :param user_id: integer + :param start_time: Datetime, time when query started. + :param tmp_table_name: table name for the CTA :return: models.Query """ - milis = int(time.time() * 1000) - query = models.Query() - query.user_id = user_id - query.database_id = database_id - if app.config.get('SQL_MAX_ROW'): - query.query_limit = app.config.get('SQL_MAX_ROW') - query.query_name = str(milis) - query.query_text = sql + # TODO(bkyryliuk): provide a way for the user to name the query. # TODO(bkyryliuk): run explain query to derive the tables and fill in the # table_ids # TODO(bkyryliuk): check the user permissions - query.start_time = milis - query.query_status = models.QueryStatus.IN_PROGRESS + limit = app.config.get('SQL_MAX_ROW', None) + if not tmp_table_name: + tmp_table_name = 'tmp.{}_table_{}'.format(user_id, start_time) + query = models.Query( + user_id=user_id, + database_id=database_id, + limit=limit, + name='{}'.format(start_time), + sql=sql, + start_time=start_time, + tmp_table_name=tmp_table_name, + status=models.QueryStatus.IN_PROGRESS + ) return query -def create_table_as(sql, table_name): +def is_query_select(sql): + try: + return sqlparse.parse(sql)[0].get_type() == 'SELECT' + # Capture sqlparse exceptions, worker shouldn't fail here. + except Exception: + # TODO(bkyryliuk): add logging here. + return False + + +# if sqlparse provides the stream of tokens but don't provide the API +# to access the table names, more on it: +# https://groups.google.com/forum/#!topic/sqlparse/sL2aAi6dSJU +# https://github.com/andialbrecht/sqlparse/blob/master/examples/ +# extract_table_names.py +# +# Another approach would be to run the EXPLAIN on the sql statement: +# https://prestodb.io/docs/current/sql/explain.html +# https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Explain +def get_tables(): + """Retrieves the query names from the query.""" + # TODO(bkyryliuk): implement parsing the sql statement. + pass + + +def add_limit_to_the_query(sql, limit, eng): + # Treat as single sql statement in case of failure. + sql_statements = [sql] + try: + sql_statements = [s for s in sqlparse.split(sql) if s] + except Exception as e: + logging.info( + "Statement " + sql + "failed to be transformed to have the limit " + "with the exception" + e.message) + return sql + if len(sql_statements) == 1 and is_query_select(sql): + qry = select('*').select_from( + TextAsFrom(text(sql_statements[0]), ['*']).alias( + 'inner_qry')).limit(limit) + sql_statement = str(qry.compile( + eng, compile_kwargs={"literal_binds": True})) + return sql_statement + return sql + + +# create table works only for the single statement. +def create_table_as(sql, table_name, override=False): """Reformats the query into the create table as query. + Works only for the single select SQL statements, in all other cases + the sql query is not modified. :param sql: string, sql query that will be executed :param table_name: string, will contain the results of the query execution + :param override, boolean, table table_name will be dropped if true :return: string, create table as query """ - return "CREATE TABLE %s AS %s" % (table_name, sql) + # TODO(bkyryliuk): drop table if allowed, check the namespace and + # the permissions. + # Treat as single sql statement in case of failure. + sql_statements = [sql] + try: + # Filter out empty statements. + sql_statements = [s for s in sqlparse.split(sql) if s] + except Exception as e: + logging.info( + "Statement " + sql + "failed to be transformed as create table as " + "with the exception" + e.message) + return sql + if len(sql_statements) == 1 and is_query_select(sql): + updated_sql = '' + # TODO(bkyryliuk): use sqlalchemy statements for the + # the drop and create operations. + if override: + updated_sql = 'DROP TABLE IF EXISTS {};\n'.format(table_name) + updated_sql += "CREATE TABLE %s AS %s" % ( + table_name, sql_statements[0]) + return updated_sql + return sql def get_session(): @@ -52,8 +128,9 @@ def get_session(): return scoped_session(sessionmaker( autocommit=False, autoflush=False, bind=engine)) + @celery_app.task -def get_sql_results(database_id, sql, user_id): +def get_sql_results(database_id, sql, user_id, tmp_table_name="", schema=None): """Executes the sql query returns the results. :param database_id: integer @@ -63,70 +140,92 @@ def get_sql_results(database_id, sql, user_id): """ # Create a separate session, reusing the db.session leads to the # concurrency issues. - query_session = get_session() + session = get_session() try: - db_to_query = query_session.query(models.Database).filter_by( - id=database_id).first() + db_to_query = ( + session.query(models.Database).filter_by(id=database_id).first() + ) except Exception as e: - return json.dumps({'msg': str(e)}) - + return { + 'error': utils.error_msg_from_exception(e), + 'success': False + } if not db_to_query: - return json.dumps( - {'msg': "Database with id {0} is missing.".format(database_id)}) - - query = init_query(database_id, sql, user_id) - query_session.add(query) - query_session.commit() - - query_result, success = get_sql_results_internal(sql, db_to_query) + return { + 'error': "Database with id {0} is missing.".format(database_id), + 'success': False + } - query.end_time = int(time.time() * 1000) - if success: - query.query_status = models.QueryStatus.FINISHED + query = init_query( + database_id, sql, user_id, datetime.now(), tmp_table_name) + session.add(query) + session.commit() + query_result = get_sql_results_as_dict( + db_to_query, sql, query.tmp_table_name, schema=schema) + query.end_time = datetime.now() + if query_result['success']: + query.status = models.QueryStatus.FINISHED # TODO(bkyryliuk): fill in query.tmp_table_name else: - query.query_status = models.QueryStatus.FAILED - - query_session.commit() + query.status = models.QueryStatus.FAILED + session.commit() # TODO(bkyryliuk): return the tmp table / query_id return query_result # TODO(bkyryliuk): merge the changes made in the carapal first # before merging this PR. -def get_sql_results_internal(sql, db_to_query): - """Get the SQL query resulst from the give session and db connection. +def get_sql_results_as_dict(db_to_query, sql, tmp_table_name, schema=None): + """Get the SQL query results from the give session and db connection. :param sql: string, query that will be executed - :param db_to_query: models.Database to query + :param db_to_query: models.Database to query, cannot be None + :param tmp_table_name: name of the table for CTA :return: (dataframe, boolean), results and the status """ + eng = db_to_query.get_sqla_engine(schema=schema) + sql = sql.strip().strip(';') + # TODO(bkyryliuk): fix this case for multiple statements + if app.config.get('SQL_MAX_ROW'): + sql = add_limit_to_the_query( + sql, app.config.get("SQL_MAX_ROW"), eng) + + cta_used = False + if (app.config.get('SQL_SELECT_AS_CTA') and + db_to_query.select_as_create_table_as and is_query_select(sql)): + # TODO(bkyryliuk): figure out if the query is select query. + sql = create_table_as(sql, tmp_table_name) + cta_used = True + + if cta_used: + try: + eng.execute(sql) + return { + 'tmp_table': tmp_table_name, + 'success': True + } + except Exception as e: + return { + 'error': utils.error_msg_from_exception(e), + 'success': False + } + + # otherwise run regular SQL query. + # TODO(bkyryliuk): rewrite into eng.execute as queries different from + # select should be permitted too. try: - eng = db_to_query.get_sqla_engine() - if app.config.get('SQL_MAX_ROW'): - sql = sql.strip().strip(';') - qry = ( - select('*') - .select_from(TextAsFrom(text(sql), ['*']).alias('inner_qry')) - .limit(app.config.get('SQL_MAX_ROW')) - ) - sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) - if db_to_query.select_as_create_table_as: - sql = sql.strip().strip(';') - # TODO(bkyryliuk): figure out if the query is select query. - sql = create_table_as( - sql, "query_" + str(int(round(time.time() * 1000)))) - - df = pd.read_sql_query(sql=sql, con=eng) - # TODO(bkyryliuk): refactore the output to be json instead of html - data = { + df = db_to_query.get_df(sql, schema) + df = df.fillna(0) + return { 'columns': [c for c in df.columns], 'data': df.to_dict(orient='records'), + 'success': True } - return json.dumps(data, allow_nan=False), True + except Exception as e: - if hasattr(e, 'message'): - return json.dumps({'msg': e.message}), False - else: - return json.dumps({'msg': str(e)}), False + return { + 'error': utils.error_msg_from_exception(e), + 'success': False + } + diff --git a/caravel/views.py b/caravel/views.py index da306510e64b3..ab68b73340612 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -32,7 +32,9 @@ from wtforms.validators import ValidationError import caravel -from caravel import appbuilder, db, models, viz, utils, app, sm, ascii_art +from caravel import ( + appbuilder, db, models, viz, utils, app, sm, ascii_art, tasks +) config = app.config log_this = models.Log.log_this @@ -1325,12 +1327,9 @@ def theme(self): @log_this def sql_json(self): """Runs arbitrary sql and returns and json""" - session = db.session() - limit = 1000 sql = request.form.get('sql') database_id = request.form.get('database_id') schema = request.form.get('schema') - mydb = session.query(models.Database).filter_by(id=database_id).first() if ( not self.can_access( @@ -1338,41 +1337,18 @@ def sql_json(self): raise utils.CaravelSecurityException(_( "This view requires the `all_datasource_access` permission")) - error_msg = "" - if not mydb: - error_msg = "The database selected doesn't seem to exist" - else: - eng = mydb.get_sqla_engine() - if limit: - sql = sql.strip().strip(';') - qry = ( - select('*') - .select_from(TextAsFrom(text(sql), ['*']).alias('inner_qry')) - .limit(limit) - ) - sql = str(qry.compile(eng, compile_kwargs={"literal_binds": True})) - try: - df = mydb.get_df(sql, schema) - df = df.fillna(0) # TODO make sure NULL - except Exception as e: - error_msg = utils.error_msg_from_exception(e) - - session.commit() - if error_msg: - return Response( - json.dumps({ - 'error': error_msg, - }), - status=500, - mimetype="application/json") - else: - data = { - 'columns': [c for c in df.columns], - 'data': df.to_dict(orient='records'), - 'ydata_tpe.to_dict': { - k: '{}'.format(v) for k, v in df.dtypes.to_dict().items()}, - } - return json.dumps(data, default=utils.json_int_dttm_ser, allow_nan=False) + data = tasks.get_sql_results(database_id, sql, g.user.get_id(), + schema=schema) + if 'error' in data: + return Response( + json.dumps(data), + status=500, + mimetype="application/json") + if 'tmp_table' in data: + # TODO(bkyryliuk) implement retrieving the data from tmp table. + return None + return json.dumps( + data, default=utils.json_int_dttm_ser, allow_nan=False) @has_access @expose("/refresh_datasources/") diff --git a/tests/caravel_test_config.py b/tests/caravel_test_config.py index 203a5f5025d33..03f6b5aede723 100644 --- a/tests/caravel_test_config.py +++ b/tests/caravel_test_config.py @@ -2,16 +2,20 @@ AUTH_USER_REGISTRATION_ROLE = 'alpha' SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/caravel_unittests.db' +# MySQL connection string for unit tests: +# SQLALCHEMY_DATABASE_URI = 'mysql://root:@localhost/caravel_db' DEBUG = True CARAVEL_WEBSERVER_PORT = 8081 # Allowing SQLALCHEMY_DATABASE_URI to be defined as an env var for # continuous integration if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ: - SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI') + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'CARAVEL__SQLALCHEMY_DATABASE_URI') SQL_CELERY_DB_FILE_PATH = '/tmp/celerydb.sqlite' SQL_CELERY_RESULTS_DB_FILE_PATH = '/tmp/celery_results.sqlite' +SQL_SELECT_AS_CTA = True class CeleryConfig(object): @@ -21,3 +25,4 @@ class CeleryConfig(object): CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} CONCURRENCY = 1 CELERY_CONFIG = CeleryConfig + diff --git a/tests/celery_tests.py b/tests/celery_tests.py index 72c5e6154f8a3..ff2f23e657fe9 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -1,13 +1,12 @@ -'''Unit tests for Caravel Celery worker''' +"""Unit tests for Caravel Celery worker""" +import datetime import imp -import json import subprocess import os +import pandas as pd import time import unittest -# Set environ before caravel module is imported. -os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' import caravel from caravel import app, appbuilder, db, models, tasks, utils @@ -23,6 +22,107 @@ class CeleryConfig(object): cli = imp.load_source('cli', BASE_DIR + '/bin/caravel') +class UtilityFunctionTests(unittest.TestCase): + def test_init_query(self): + date = datetime.datetime(year=2016, month=11, day=6) + query = tasks.init_query(1, "sql", 2, date, 'tmp') + self.assertEqual(1, query.database_id) + self.assertEqual("sql", query.sql) + self.assertEqual(2, query.user_id) + self.assertEqual(models.QueryStatus.IN_PROGRESS, query.status) + self.assertEqual(date, query.start_time) + self.assertEqual(str(date), query.name) + self.assertEqual('tmp', query.tmp_table_name) + + def test_create_table_as(self): + select_query = "SELECT * FROM outer_space;" + updated_select_query = tasks.create_table_as(select_query, "tmp") + self.assertEqual( + "CREATE TABLE tmp AS SELECT * FROM outer_space;", + updated_select_query) + + updated_select_query_with_drop = tasks.create_table_as( + select_query, "tmp", override=True) + self.assertEqual( + "DROP TABLE IF EXISTS tmp;\n" + "CREATE TABLE tmp AS SELECT * FROM outer_space;", + updated_select_query_with_drop) + + select_query_no_semicolon = "SELECT * FROM outer_space" + updated_select_query_no_semicolon = tasks.create_table_as( + select_query_no_semicolon, "tmp") + self.assertEqual( + "CREATE TABLE tmp AS SELECT * FROM outer_space", + updated_select_query_no_semicolon) + + incorrect_query = "SMTH WRONG SELECT * FROM outer_space" + updated_incorrect_query = tasks.create_table_as(incorrect_query, "tmp") + self.assertEqual(incorrect_query, updated_incorrect_query) + + insert_query = "INSERT INTO stomach VALUES (beer, chips);" + updated_insert_query = tasks.create_table_as(insert_query, "tmp") + self.assertEqual(insert_query, updated_insert_query) + + multi_line_query = ( + "SELECT * FROM planets WHERE\n" + "Luke_Father = 'Darth Vader';") + updated_multi_line_query = tasks.create_table_as( + multi_line_query, "tmp") + expected_updated_multi_line_query = ( + "CREATE TABLE tmp AS SELECT * FROM planets WHERE\n" + "Luke_Father = 'Darth Vader';") + self.assertEqual( + expected_updated_multi_line_query, + updated_multi_line_query) + + updated_multi_line_query_with_drop = tasks.create_table_as( + multi_line_query, "tmp", override=True) + expected_updated_multi_line_query_with_drop = ( + "DROP TABLE IF EXISTS tmp;\n" + "CREATE TABLE tmp AS SELECT * FROM planets WHERE\n" + "Luke_Father = 'Darth Vader';") + self.assertEqual( + expected_updated_multi_line_query_with_drop, + updated_multi_line_query_with_drop) + + delete_query = "DELETE FROM planet WHERE name = 'Earth'" + updated_delete_query = tasks.create_table_as(delete_query, "tmp") + self.assertEqual(delete_query, updated_delete_query) + + create_table_as = ( + "CREATE TABLE pleasure AS SELECT chocolate FROM lindt_store;\n") + updated_create_table_as = tasks.create_table_as( + create_table_as, "tmp") + self.assertEqual(create_table_as, updated_create_table_as) + + sql_procedure = ( + "CREATE PROCEDURE MyMarriage\n " + "BrideGroom Male (25) ,\n " + "Bride Female(20) AS\n " + "BEGIN\n " + "SELECT Bride FROM ukraine_ Brides\n " + "WHERE\n " + "FatherInLaw = 'Millionaire' AND Count(Car) > 20\n" + " AND HouseStatus ='ThreeStoreyed'\n" + " AND BrideEduStatus IN " + "(B.TECH ,BE ,Degree ,MCA ,MiBA)\n " + "AND Having Brothers= Null AND Sisters =Null" + ) + updated_sql_procedure = tasks.create_table_as(sql_procedure, "tmp") + self.assertEqual(sql_procedure, updated_sql_procedure) + + multiple_statements = """ + DROP HUSBAND; + SELECT * FROM politicians WHERE clue > 0; + INSERT INTO MyCarShed VALUES('BUGATTI'); + SELECT standard_disclaimer, witty_remark FROM company_requirements; + select count(*) from developer_brain; + """ + updated_multiple_statements = tasks.create_table_as( + multiple_statements, "tmp") + self.assertEqual(multiple_statements, updated_multiple_statements) + + class CeleryTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): super(CeleryTestCase, self).__init__(*args, **kwargs) @@ -71,99 +171,238 @@ def setUp(self): def tearDown(self): pass + def test_add_limit_to_the_query(self): + query_session = tasks.get_session() + db_to_query = query_session.query(models.Database).filter_by( + id=1).first() + eng = db_to_query.get_sqla_engine() + + select_query = "SELECT * FROM outer_space;" + updated_select_query = tasks.add_limit_to_the_query( + select_query, 100, eng) + # Different DB engines have their own spacing while compiling + # the queries, that's why ' '.join(query.split()) is used. + # In addition some of the engines do not include OFFSET 0. + self.assertTrue( + "SELECT * FROM (SELECT * FROM outer_space;) AS inner_qry " + "LIMIT 100" in ' '.join(updated_select_query.split()) + ) + + select_query_no_semicolon = "SELECT * FROM outer_space" + updated_select_query_no_semicolon = tasks.add_limit_to_the_query( + select_query_no_semicolon, 100, eng) + self.assertTrue( + "SELECT * FROM (SELECT * FROM outer_space) AS inner_qry " + "LIMIT 100" in + ' '.join(updated_select_query_no_semicolon.split()) + ) + + incorrect_query = "SMTH WRONG SELECT * FROM outer_space" + updated_incorrect_query = tasks.add_limit_to_the_query( + incorrect_query, 100, eng) + self.assertEqual(incorrect_query, updated_incorrect_query) + + insert_query = "INSERT INTO stomach VALUES (beer, chips);" + updated_insert_query = tasks.add_limit_to_the_query( + insert_query, 100, eng) + self.assertEqual(insert_query, updated_insert_query) + + multi_line_query = ( + "SELECT * FROM planets WHERE\n Luke_Father = 'Darth Vader';" + ) + updated_multi_line_query = tasks.add_limit_to_the_query( + multi_line_query, 100, eng) + self.assertTrue( + "SELECT * FROM (SELECT * FROM planets WHERE " + "Luke_Father = 'Darth Vader';) AS inner_qry LIMIT 100" in + ' '.join(updated_multi_line_query.split()) + ) + + delete_query = "DELETE FROM planet WHERE name = 'Earth'" + updated_delete_query = tasks.add_limit_to_the_query( + delete_query, 100, eng) + self.assertEqual(delete_query, updated_delete_query) + + create_table_as = ( + "CREATE TABLE pleasure AS SELECT chocolate FROM lindt_store;\n") + updated_create_table_as = tasks.add_limit_to_the_query( + create_table_as, 100, eng) + self.assertEqual(create_table_as, updated_create_table_as) + + sql_procedure = ( + "CREATE PROCEDURE MyMarriage\n " + "BrideGroom Male (25) ,\n " + "Bride Female(20) AS\n " + "BEGIN\n " + "SELECT Bride FROM ukraine_ Brides\n " + "WHERE\n " + "FatherInLaw = 'Millionaire' AND Count(Car) > 20\n" + " AND HouseStatus ='ThreeStoreyed'\n" + " AND BrideEduStatus IN " + "(B.TECH ,BE ,Degree ,MCA ,MiBA)\n " + "AND Having Brothers= Null AND Sisters = Null" + ) + updated_sql_procedure = tasks.add_limit_to_the_query( + sql_procedure, 100, eng) + self.assertEqual(sql_procedure, updated_sql_procedure) + def test_run_async_query_delay_get(self): + main_db = db.session.query(models.Database).filter_by( + database_name="main").first() + eng = main_db.get_sqla_engine() + + # Case 1. # DB #0 doesn't exist. result1 = tasks.get_sql_results.delay( - 0, 'SELECT * FROM dontexist', 1).get() - expected_result1 = {'msg': 'Database with id 0 is missing.'} - self.assertEqual(expected_result1, json.loads(result1)) + 0, 'SELECT * FROM dontexist', 1, tmp_table_name='tmp_1_1').get() + expected_result1 = { + 'error': 'Database with id 0 is missing.', + 'success': False + } + self.assertEqual( + sorted(expected_result1.items()), + sorted(result1.items()) + ) session1 = db.create_scoped_session() query1 = session1.query(models.Query).filter_by( - query_text='SELECT * FROM dontexist').first() + sql='SELECT * FROM dontexist').first() session1.close() self.assertIsNone(query1) + # Case 2. session2 = db.create_scoped_session() query2 = session2.query(models.Query).filter_by( - query_text='SELECT * FROM dontexist1').first() - self.assertEqual(models.QueryStatus.FAILED, query2.query_status) + sql='SELECT * FROM dontexist1').first() + self.assertEqual(models.QueryStatus.FAILED, query2.status) session2.close() result2 = tasks.get_sql_results.delay( - 1, 'SELECT * FROM dontexist1', 1).get() - self.assertTrue('msg' in result2) + 1, 'SELECT * FROM dontexist1', 1, tmp_table_name='tmp_2_1').get() + self.assertTrue('error' in result2) session2 = db.create_scoped_session() query2 = session2.query(models.Query).filter_by( - query_text='SELECT * FROM dontexist1').first() - self.assertEqual(models.QueryStatus.FAILED, query2.query_status) + sql='SELECT * FROM dontexist1').first() + self.assertEqual(models.QueryStatus.FAILED, query2.status) session2.close() + # Case 3. where_query = ( "SELECT name FROM ab_permission WHERE name='can_select_star'") - result3 = tasks.get_sql_results.delay(1, where_query, 1).get() + result3 = tasks.get_sql_results.delay( + 1, where_query, 1, tmp_table_name='tmp_3_1').get() expected_result3 = { - 'columns': ['name'], 'data': [{'name': 'can_select_star'}]} + 'tmp_table': 'tmp_3_1', + 'success': True + } self.assertEqual( sorted(expected_result3.items()), - sorted(json.loads(result3).items())) + sorted(result3.items()) + ) session3 = db.create_scoped_session() query3 = session3.query(models.Query).filter_by( - query_text=where_query).first() - self.assertEqual(models.QueryStatus.FINISHED, query3.query_status) + sql=where_query).first() session3.close() + df3 = pd.read_sql_query(sql="SELECT * FROM tmp_3_1", con=eng) + data3 = df3.to_dict(orient='records') + self.assertEqual(models.QueryStatus.FINISHED, query3.status) + self.assertEqual([{'name': 'can_select_star'}], data3) + # Case 4. result4 = tasks.get_sql_results.delay( - 1, 'SELECT * FROM ab_permission WHERE id=666', 1).get() - expected_result4 = {'columns': ['id', 'name'], 'data': []} - self.assertEqual(expected_result4, json.loads(result4)) + 1, 'SELECT * FROM ab_permission WHERE id=666', 1, + tmp_table_name='tmp_4_1').get() + expected_result4 = { + 'tmp_table': 'tmp_4_1', + 'success': True + } + self.assertEqual( + sorted(expected_result4.items()), + sorted(result4.items()) + ) session4 = db.create_scoped_session() query4 = session4.query(models.Query).filter_by( - query_text='SELECT * FROM ab_permission WHERE id=666').first() - self.assertEqual(models.QueryStatus.FINISHED, query4.query_status) + sql='SELECT * FROM ab_permission WHERE id=666').first() session4.close() + df4 = pd.read_sql_query(sql="SELECT * FROM tmp_4_1", con=eng) + data4 = df4.to_dict(orient='records') + self.assertEqual(models.QueryStatus.FINISHED, query4.status) + self.assertEqual([], data4) + + # Case 5. + # Return the data directly if DB select_as_create_table_as is False. + main_db.select_as_create_table_as = False + db.session.commit() + result5 = tasks.get_sql_results.delay( + 1, where_query, 1, tmp_table_name='tmp_5_1').get() + expected_result5 = { + 'columns': ['name'], + 'data': [{'name': 'can_select_star'}], + 'success': True + } + self.assertEqual( + sorted(expected_result5.items()), + sorted(result5.items()) + ) def test_run_async_query_delay(self): celery_task1 = tasks.get_sql_results.delay( - 0, 'SELECT * FROM dontexist', 1) + 0, 'SELECT * FROM dontexist', 1, tmp_table_name='tmp_1_2') celery_task2 = tasks.get_sql_results.delay( - 1, 'SELECT * FROM dontexist1', 1) + 1, 'SELECT * FROM dontexist1', 1, tmp_table_name='tmp_2_2') where_query = ( "SELECT name FROM ab_permission WHERE name='can_select_star'") - celery_task3 = tasks.get_sql_results.delay(1, where_query, 1) + celery_task3 = tasks.get_sql_results.delay( + 1, where_query, 1, tmp_table_name='tmp_3_2') celery_task4 = tasks.get_sql_results.delay( - 1, 'SELECT * FROM ab_permission WHERE id=666', 1) + 1, 'SELECT * FROM ab_permission WHERE id=666', 1, + tmp_table_name='tmp_4_2') - time.sleep(2) + time.sleep(1) # DB #0 doesn't exist. - expected_result1 = {'msg': 'Database with id 0 is missing.'} - self.assertEqual(expected_result1, json.loads(celery_task1.get())) + expected_result1 = { + 'error': 'Database with id 0 is missing.', + 'success': False + } + self.assertEqual( + sorted(expected_result1.items()), + sorted(celery_task1.get().items()) + ) session2 = db.create_scoped_session() query2 = session2.query(models.Query).filter_by( - query_text='SELECT * FROM dontexist1').first() - self.assertEqual(models.QueryStatus.FAILED, query2.query_status) - self.assertTrue('msg' in celery_task2.get()) + sql='SELECT * FROM dontexist1').first() + self.assertEqual(models.QueryStatus.FAILED, query2.status) + self.assertTrue('error' in celery_task2.get()) expected_result3 = { - 'columns': ['name'], 'data': [{'name': 'can_select_star'}]} + 'tmp_table': 'tmp_3_2', + 'success': True + } self.assertEqual( sorted(expected_result3.items()), - sorted(json.loads(celery_task3.get()).items())) - expected_result4 = {'columns': ['id', 'name'], 'data': []} - self.assertEqual(expected_result4, json.loads(celery_task4.get())) + sorted(celery_task3.get().items()) + ) + expected_result4 = { + 'tmp_table': 'tmp_4_2', + 'success': True + } + self.assertEqual( + sorted(expected_result4.items()), + sorted(celery_task4.get().items()) + ) session = db.create_scoped_session() query1 = session.query(models.Query).filter_by( - query_text='SELECT * FROM dontexist').first() + sql='SELECT * FROM dontexist').first() self.assertIsNone(query1) query2 = session.query(models.Query).filter_by( - query_text='SELECT * FROM dontexist1').first() - self.assertEqual(models.QueryStatus.FAILED, query2.query_status) + sql='SELECT * FROM dontexist1').first() + self.assertEqual(models.QueryStatus.FAILED, query2.status) query3 = session.query(models.Query).filter_by( - query_text=where_query).first() - self.assertEqual(models.QueryStatus.FINISHED, query3.query_status) + sql=where_query).first() + self.assertEqual(models.QueryStatus.FINISHED, query3.status) query4 = session.query(models.Query).filter_by( - query_text='SELECT * FROM ab_permission WHERE id=666').first() - self.assertEqual(models.QueryStatus.FINISHED, query4.query_status) + sql='SELECT * FROM ab_permission WHERE id=666').first() + self.assertEqual(models.QueryStatus.FINISHED, query4.status) session.close() diff --git a/tests/core_tests.py b/tests/core_tests.py index 77212094859cb..08623ab34bb21 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -17,16 +17,20 @@ from flask_appbuilder.security.sqla import models as ab_models import caravel -from caravel import app, db, models, utils, appbuilder, tasks -from caravel.models import DruidCluster +from caravel import app, db, models, utils, appbuilder +from caravel.models import DruidCluster, DruidDatasource os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' -app.config['TESTING'] = True +# Disable celery. +app.config['CELERY_CONFIG'] = None app.config['CSRF_ENABLED'] = False +app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True app.config['SECRET_KEY'] = 'thisismyscretkey' +app.config['SQL_SELECT_AS_CTA'] = False +app.config['TESTING'] = True app.config['WTF_CSRF_ENABLED'] = False -app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True + BASE_DIR = app.config.get("BASE_DIR") cli = imp.load_source('cli', BASE_DIR + "/bin/caravel") @@ -242,7 +246,10 @@ def test_add_slice_redirect_to_sqla(self, username='admin'): self.login(username=username) url = '/slicemodelview/add' resp = self.client.get(url, follow_redirects=True) - assert "Click on a table link to create a Slice" in resp.data.decode('utf-8') + assert ( + "Click on a table link to create a Slice" in + resp.data.decode('utf-8') + ) def test_add_slice_redirect_to_druid(self, username='admin'): datasource = DruidDatasource( @@ -254,7 +261,10 @@ def test_add_slice_redirect_to_druid(self, username='admin'): self.login(username=username) url = '/slicemodelview/add' resp = self.client.get(url, follow_redirects=True) - assert "Click on a datasource link to create a Slice" in resp.data.decode('utf-8') + assert ( + "Click on a datasource link to create a Slice" + in resp.data.decode('utf-8') + ) db.session.delete(datasource) db.session.commit() From 86794b2828269a31e1f665a46533cc1deb018e5e Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Mon, 8 Aug 2016 17:46:09 -0700 Subject: [PATCH 11/14] Address the comments. --- caravel/tasks.py | 53 +++++++++++++++++------------------------------- caravel/views.py | 3 ++- caravel/viz.py | 3 ++- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/caravel/tasks.py b/caravel/tasks.py index 62a3a09f563c9..48df9a73103be 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -11,37 +11,6 @@ celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG')) -def init_query(database_id, sql, user_id, start_time, tmp_table_name): - """Initializes the models.Query object. - - :param database_id: integer - :param sql: sql query that will be executed - :param user_id: integer - :param start_time: Datetime, time when query started. - :param tmp_table_name: table name for the CTA - :return: models.Query - """ - - # TODO(bkyryliuk): provide a way for the user to name the query. - # TODO(bkyryliuk): run explain query to derive the tables and fill in the - # table_ids - # TODO(bkyryliuk): check the user permissions - limit = app.config.get('SQL_MAX_ROW', None) - if not tmp_table_name: - tmp_table_name = 'tmp.{}_table_{}'.format(user_id, start_time) - query = models.Query( - user_id=user_id, - database_id=database_id, - limit=limit, - name='{}'.format(start_time), - sql=sql, - start_time=start_time, - tmp_table_name=tmp_table_name, - status=models.QueryStatus.IN_PROGRESS - ) - return query - - def is_query_select(sql): try: return sqlparse.parse(sql)[0].get_type() == 'SELECT' @@ -156,8 +125,25 @@ def get_sql_results(database_id, sql, user_id, tmp_table_name="", schema=None): 'success': False } - query = init_query( - database_id, sql, user_id, datetime.now(), tmp_table_name) + # TODO(bkyryliuk): provide a way for the user to name the query. + # TODO(bkyryliuk): run explain query to derive the tables and fill in the + # table_ids + # TODO(bkyryliuk): check the user permissions + # TODO(bkyryliuk): store the tab name in the query model + limit = app.config.get('SQL_MAX_ROW', None) + start_time = datetime.now() + if not tmp_table_name: + tmp_table_name = 'tmp.{}_table_{}'.format(user_id, start_time) + query = models.Query( + user_id=user_id, + database_id=database_id, + limit=limit, + name='{}'.format(start_time), + sql=sql, + start_time=start_time, + tmp_table_name=tmp_table_name, + status=models.QueryStatus.IN_PROGRESS + ) session.add(query) session.commit() query_result = get_sql_results_as_dict( @@ -165,7 +151,6 @@ def get_sql_results(database_id, sql, user_id, tmp_table_name="", schema=None): query.end_time = datetime.now() if query_result['success']: query.status = models.QueryStatus.FINISHED - # TODO(bkyryliuk): fill in query.tmp_table_name else: query.status = models.QueryStatus.FAILED session.commit() diff --git a/caravel/views.py b/caravel/views.py index ab68b73340612..a23942f2847d6 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -1345,7 +1345,8 @@ def sql_json(self): status=500, mimetype="application/json") if 'tmp_table' in data: - # TODO(bkyryliuk) implement retrieving the data from tmp table. + # TODO(bkyryliuk): add query id to the response and implement the + # endpoint to poll the status and results. return None return json.dumps( data, default=utils.json_int_dttm_ser, allow_nan=False) diff --git a/caravel/viz.py b/caravel/viz.py index d85fc83ba6ba9..83df0c4729fd0 100755 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -284,7 +284,8 @@ def get_json(self): cached_data = cached_data.decode('utf-8') payload = json.loads(cached_data) except Exception as e: - logging.error("Error reading cache: " + str(e)) + logging.error("Error reading cache: " + + utils.error_msg_from_exception(e)) payload = None logging.info("Serving from cache") From 7a30dee5397066e0855b031aa82f0d618226aaec Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Mon, 8 Aug 2016 17:48:53 -0700 Subject: [PATCH 12/14] Add trailing commas --- caravel/tasks.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/caravel/tasks.py b/caravel/tasks.py index 48df9a73103be..dc8bd604c4104 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -117,12 +117,12 @@ def get_sql_results(database_id, sql, user_id, tmp_table_name="", schema=None): except Exception as e: return { 'error': utils.error_msg_from_exception(e), - 'success': False + 'success': False, } if not db_to_query: return { 'error': "Database with id {0} is missing.".format(database_id), - 'success': False + 'success': False, } # TODO(bkyryliuk): provide a way for the user to name the query. @@ -142,7 +142,7 @@ def get_sql_results(database_id, sql, user_id, tmp_table_name="", schema=None): sql=sql, start_time=start_time, tmp_table_name=tmp_table_name, - status=models.QueryStatus.IN_PROGRESS + status=models.QueryStatus.IN_PROGRESS, ) session.add(query) session.commit() @@ -187,12 +187,12 @@ def get_sql_results_as_dict(db_to_query, sql, tmp_table_name, schema=None): eng.execute(sql) return { 'tmp_table': tmp_table_name, - 'success': True + 'success': True, } except Exception as e: return { 'error': utils.error_msg_from_exception(e), - 'success': False + 'success': False, } # otherwise run regular SQL query. @@ -204,13 +204,13 @@ def get_sql_results_as_dict(db_to_query, sql, tmp_table_name, schema=None): return { 'columns': [c for c in df.columns], 'data': df.to_dict(orient='records'), - 'success': True + 'success': True, } except Exception as e: return { 'error': utils.error_msg_from_exception(e), - 'success': False + 'success': False, } From fc94502372c63231dc3970acd7f55eb3cffebfaf Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Mon, 8 Aug 2016 17:51:11 -0700 Subject: [PATCH 13/14] Remove the init_query test. --- caravel/tasks.py | 3 +++ tests/celery_tests.py | 11 ----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/caravel/tasks.py b/caravel/tasks.py index dc8bd604c4104..c48e66997456a 100644 --- a/caravel/tasks.py +++ b/caravel/tasks.py @@ -105,6 +105,8 @@ def get_sql_results(database_id, sql, user_id, tmp_table_name="", schema=None): :param database_id: integer :param sql: string, query that will be executed :param user_id: integer + :param tmp_table_name: name of the table for CTA + :param schema: string, name of the schema (used in presto) :return: dataframe, query result """ # Create a separate session, reusing the db.session leads to the @@ -166,6 +168,7 @@ def get_sql_results_as_dict(db_to_query, sql, tmp_table_name, schema=None): :param sql: string, query that will be executed :param db_to_query: models.Database to query, cannot be None :param tmp_table_name: name of the table for CTA + :param schema: string, name of the schema (used in presto) :return: (dataframe, boolean), results and the status """ eng = db_to_query.get_sqla_engine(schema=schema) diff --git a/tests/celery_tests.py b/tests/celery_tests.py index ff2f23e657fe9..e88ae0fca1c5b 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -23,17 +23,6 @@ class CeleryConfig(object): class UtilityFunctionTests(unittest.TestCase): - def test_init_query(self): - date = datetime.datetime(year=2016, month=11, day=6) - query = tasks.init_query(1, "sql", 2, date, 'tmp') - self.assertEqual(1, query.database_id) - self.assertEqual("sql", query.sql) - self.assertEqual(2, query.user_id) - self.assertEqual(models.QueryStatus.IN_PROGRESS, query.status) - self.assertEqual(date, query.start_time) - self.assertEqual(str(date), query.name) - self.assertEqual('tmp', query.tmp_table_name) - def test_create_table_as(self): select_query = "SELECT * FROM outer_space;" updated_select_query = tasks.create_table_as(select_query, "tmp") From 9fc0e84057f07be08ddc967eddeb549985eedb52 Mon Sep 17 00:00:00 2001 From: Bogdan Kyryliuk Date: Tue, 9 Aug 2016 18:36:48 -0700 Subject: [PATCH 14/14] Handle 'undefined' schema case --- caravel/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caravel/views.py b/caravel/views.py index a23942f2847d6..b7a206b2c660c 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -1062,7 +1062,7 @@ def activity_per_day(self): @expose("/tables//") def tables(self, db_id, schema): """endpoint to power the calendar heatmap on the welcome page""" - schema = None if schema == 'null' else schema + schema = None if schema in ('null', 'undefined') else schema database = ( db.session .query(models.Database) @@ -1231,7 +1231,7 @@ def sql(self, database_id): @expose("/table////") @log_this def table(self, database_id, table_name, schema): - schema = None if schema == 'null' else schema + schema = None if schema in ('null', 'undefined') else schema mydb = db.session.query(models.Database).filter_by(id=database_id).one() cols = [] t = mydb.get_columns(table_name, schema)