Schema Viewer Drawer (getredash#3291)

* Process extra column metadata for a few sql-based data sources.

* Add Table and Column metadata tables.

* Periodically update table and column schema tables in a celery task.

* Fetching schema returns data from table and column metadata tables.

* Add tests for backend changes.

* Front-end shows extra table metadata and uses new schema response.

* Delete datasource schema data when deleting a data source.

* Process and store data source schema when a data source is first created or after a migration.

* Tables should have a unique name per datasource.

* Addressing review comments.

* Update migration file for mixins.

* Appease PEP8

* Upgrade migration file for rebase.

* Cascade delete.

* Adding org_id

* Remove redundant column and table prefixes.

* Non-existing tables and columns should be filtered out on the server side not client side.

* Fetching table samples should be optional and should happen in a separate task per table.

* Allow users to force a schema refresh.

* Use updated_at to help prune old schema metadata periodically.

* Using settings.SCHEMAS_REFRESH_QUEUE

* fix for getredash#2426 test

* more stable test_interactive_new

* Closes #927, #928: Schema refresh improvements.

* Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936)

* Speed up schema fetch requests with fewer postgres queries.

* Add column metadata to Athena glue processing.

* Fix bug assuming 'metadata' exists for every table.

* Closes #939: Persisted, existing table metadata should be updated.

* Sample processing should be rate-limited.

* Add cli command for refreshing data samples.

* Schema refreshes should not overwrite column 'example' field.

* refresh_samples() should filter tables_to_sample on the datasource's id being sampled

* Correctly wrap long text in schema drawer.

Co-authored-by: Alison <[email protected]>

client/app/assets/less/ant.less

@@ -14,6 +14,7 @@
 @import '~antd/lib/radio/style/index';
 @import '~antd/lib/time-picker/style/index';
 @import '~antd/lib/pagination/style/index';
+@import '~antd/lib/drawer/style/index';
 @import '~antd/lib/table/style/index';
 @import '~antd/lib/popover/style/index';
 @import '~antd/lib/icon/style/index';

client/app/assets/less/inc/schema-browser.less

@@ -7,14 +7,14 @@ div.table-name {
   border-radius: @redash-radius;
   position: relative;
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     display: none;
   }
 
   &:hover {
     background: fade(@redash-gray, 10%);
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: flex;
     }
   }
@@ -36,7 +36,7 @@ div.table-name {
     background: transparent;
   }
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     color: fade(@redash-gray, 90%);
     cursor: pointer;
     position: absolute;
@@ -49,21 +49,25 @@ div.table-name {
     justify-content: center;
   }
 
+  .info {
+    right: 20px
+  }
+
   .table-open {
     padding: 0 22px 0 26px;
     overflow: hidden;
     text-overflow: ellipsis;
     white-space: nowrap;
     position: relative;
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: none;
     }
 
     &:hover {
       background: fade(@redash-gray, 10%);
 
-      .copy-to-editor {
+      .copy-to-editor, .info {
         display: flex;
       }
     }

client/app/components/proptypes.js

@@ -11,6 +11,13 @@ export const DataSource = PropTypes.shape({
   type_name: PropTypes.string,
 });
 
+export const DataSourceMetadata = PropTypes.shape({
+  key: PropTypes.number,
+  name: PropTypes.string,
+  type: PropTypes.string,
+  example: PropTypes.string,
+});
+
 export const Table = PropTypes.shape({
   columns: PropTypes.arrayOf(PropTypes.string).isRequired,
 });

client/app/components/queries/SchemaData.jsx

@@ -0,0 +1,83 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { react2angular } from 'react2angular';
+import Drawer from 'antd/lib/drawer';
+import Table from 'antd/lib/table';
+
+import { DataSourceMetadata } from '@/components/proptypes';
+
+function textWrapRenderer(text) {
+  return (
+    <div style={{ wordWrap: 'break-word', wordBreak: 'break-all' }}>
+      {text}
+    </div>
+  );
+}
+
+class SchemaData extends React.PureComponent {
+  static propTypes = {
+    show: PropTypes.bool.isRequired,
+    onClose: PropTypes.func.isRequired,
+    tableName: PropTypes.string,
+    tableMetadata: PropTypes.arrayOf(DataSourceMetadata),
+  };
+
+  static defaultProps = {
+    tableName: '',
+    tableMetadata: [],
+  };
+
+  render() {
+    const columns = [{
+      title: 'Column Name',
+      dataIndex: 'name',
+      width: 400,
+      key: 'name',
+      render: textWrapRenderer,
+    }, {
+      title: 'Column Type',
+      dataIndex: 'type',
+      width: 400,
+      key: 'type',
+      render: textWrapRenderer,
+    }];
+
+    const hasExample =
+      this.props.tableMetadata.some(columnMetadata => columnMetadata.example);
+
+    if (hasExample) {
+      columns.push({
+        title: 'Example',
+        dataIndex: 'example',
+        width: 400,
+        key: 'example',
+        render: textWrapRenderer,
+      });
+    }
+
+    return (
+      <Drawer
+        title={this.props.tableName}
+        closable={false}
+        placement="bottom"
+        height={500}
+        onClose={this.props.onClose}
+        visible={this.props.show}
+      >
+        <Table
+          dataSource={this.props.tableMetadata}
+          pagination={false}
+          scroll={{ y: 350 }}
+          size="small"
+          columns={columns}
+        />
+      </Drawer>
+    );
+  }
+}
+
+export default function init(ngModule) {
+  ngModule.component('schemaData', react2angular(SchemaData, null, []));
+}
+
+init.init = true;

client/app/components/queries/schema-browser.html

@@ -26,15 +26,24 @@
           <span title="{{table.name}}">{{table.name}}</span>
           <span ng-if="table.size !== undefined"> ({{table.size}})</span>
         </strong>
+        <i ng-if="table.hasColumnMetadata" class="fa fa-question-circle info" title="More Info" aria-hidden="true"
+          ng-click="openSchemaInfo($event, table.name, table.columns)"></i>
         <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
           ng-click="$ctrl.itemSelected($event, [table.name])"></i>
       </div>
       <div uib-collapse="table.collapsed">
-        <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column" class="table-open">{{column}}
+        <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column.key" class="table-open">
+          {{column.name}}
           <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
-            ng-click="$ctrl.itemSelected($event, [column])"></i>
+            ng-click="$ctrl.itemSelected($event, [column.name])"></i>
         </div>
       </div>
     </div>
   </div>
+  <schema-data
+    show="showSchemaInfo"
+    table-name="tableName"
+    table-metadata="tableMetadata"
+    on-close="closeSchemaInfo"
+  ></schema-data>
 </div>

client/app/components/queries/schema-browser.js

@@ -11,6 +11,17 @@ function SchemaBrowserCtrl($rootScope, $scope) {
     $scope.$broadcast('vsRepeatTrigger');
   };
 
+  $scope.showSchemaInfo = false;
+  $scope.openSchemaInfo = ($event, tableName, tableMetadata) => {
+    $scope.tableName = tableName;
+    $scope.tableMetadata = tableMetadata;
+    $scope.showSchemaInfo = true;
+    $event.stopPropagation();
+  };
+  $scope.closeSchemaInfo = () => {
+    $scope.$apply(() => { $scope.showSchemaInfo = false; });
+  };
+
   this.getSize = (table) => {
     let size = 22;
 

migrations/versions/280daa582976_.py

@@ -0,0 +1,55 @@
+"""Add column metadata and table metadata
+
+Revision ID: 280daa582976
+Revises: b8a479422596
+Create Date: 2019-01-24 18:23:53.040608
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '280daa582976'
+down_revision = 'b8a479422596'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'table_metadata',
+        sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('org_id', sa.Integer(), nullable=False),
+        sa.Column('data_source_id', sa.Integer(), nullable=False),
+        sa.Column('exists', sa.Boolean(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.Column('description', sa.String(length=4096), nullable=True),
+        sa.Column('column_metadata', sa.Boolean(), nullable=False),
+        sa.Column('sample_query', sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(['data_source_id'], ['data_sources.id'], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(['org_id'], ['organizations.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'column_metadata',
+        sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('org_id', sa.Integer(), nullable=False),
+        sa.Column('table_id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.Column('type', sa.String(length=255), nullable=True),
+        sa.Column('example', sa.String(length=4096), nullable=True),
+        sa.Column('exists', sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(['table_id'], ['table_metadata.id'], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(['org_id'], ['organizations.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def downgrade():
+    op.drop_table('column_metadata')
+    op.drop_table('table_metadata')

migrations/versions/6adb92e75691_.py

@@ -0,0 +1,25 @@
+"""Add sample_updated_at column to table_metadata
+
+Revision ID: 6adb92e75691
+Revises: 280daa582976
+Create Date: 2019-04-10 20:13:13.714589
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6adb92e75691'
+down_revision = '280daa582976'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+	op.add_column('table_metadata', sa.Column(
+		'sample_updated_at', sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade():
+    op.drop_column('table_metadata', 'sample_updated_at')

redash/cli/data_sources.py

@@ -9,6 +9,7 @@
 from redash import models
 from redash.query_runner import (get_configuration_schema_for_query_runner_type,
                                  query_runners)
+from redash.tasks import refresh_samples
 from redash.utils import json_loads
 from redash.utils.configuration import ConfigurationContainer
 
@@ -110,7 +111,7 @@ def new(name=None, type=None, options=None, organization='default'):
 
         options_obj = {}
 
-        for k, prop in schema['properties'].iteritems():
+        for k, prop in sorted(schema['properties'].iteritems()):
             required = k in schema.get('required', [])
             default_value = "<<DEFAULT_VALUE>>"
             if required:
@@ -172,6 +173,27 @@ def update_attr(obj, attr, new_value):
         setattr(obj, attr, new_value)
 
 
+@manager.command()
+@click.argument('name')
+@click.option('--org', 'organization', default='default',
+              help="The organization the user belongs to (leave blank for "
+              "'default').")
+@click.option('--count', 'num_tables', default=50,
+              help="number of tables to process data samples for")
+def refresh_data_samples(name, num_tables=50, organization='default'):
+    """Refresh table samples by data source name."""
+    try:
+        org = models.Organization.get_by_slug(organization)
+        data_source = models.DataSource.query.filter(
+            models.DataSource.name == name,
+            models.DataSource.org == org).one()
+        print("Refreshing samples for data source: {} (id={})".format(name, data_source.id))
+        refresh_samples(data_source.id, num_tables)
+    except NoResultFound:
+        print("Couldn't find data source named: {}".format(name))
+        exit(1)
+
+
 @manager.command()
 @click.argument('name')
 @click.option('--name', 'new_name', default=None,

redash/handlers/data_sources.py

@@ -6,10 +6,11 @@
 from six import text_type
 from sqlalchemy.exc import IntegrityError
 
-from redash import models
+from redash import models, settings
 from redash.handlers.base import BaseResource, get_object_or_404, require_fields
 from redash.permissions import (require_access, require_admin,
                                 require_permission, view_only)
+from redash.tasks.queries import refresh_schema
 from redash.query_runner import (get_configuration_schema_for_query_runner_type,
                                  query_runners, NotSupported)
 from redash.utils import filter_none
@@ -52,6 +53,9 @@ def post(self, data_source_id):
         data_source.name = req['name']
         models.db.session.add(data_source)
 
+        # Refresh the stored schemas when a data source is updated
+        refresh_schema.apply_async(args=(data_source.id,), queue=settings.SCHEMAS_REFRESH_QUEUE)
+
         try:
             models.db.session.commit()
         except IntegrityError as e:
@@ -127,6 +131,9 @@ def post(self):
                                                              options=config)
 
             models.db.session.commit()
+
+            # Refresh the stored schemas when a new data source is added to the list
+            refresh_schema.apply_async(args=(datasource.id,), queue=settings.SCHEMAS_REFRESH_QUEUE)
         except IntegrityError as e:
             models.db.session.rollback()
             if req['name'] in e.message:
@@ -150,9 +157,10 @@ def get(self, data_source_id):
         refresh = request.args.get('refresh') is not None
 
         response = {}
-
         try:
-            response['schema'] = data_source.get_schema(refresh)
+            if refresh:
+                refresh_schema.apply_async(args=(data_source.id,), queue=settings.SCHEMAS_REFRESH_QUEUE)
+            response['schema'] = data_source.get_schema()
         except NotSupported:
             response['error'] = {
                 'code': 1,