feat: add metastore loader config (pinterest#1134)

* feat: add metastore config

* comments

* rename

* remove session

package.json

@@ -1,6 +1,6 @@
 {
     "name": "querybook",
-    "version": "3.16.1",
+    "version": "3.17.0",
     "description": "A Big Data Webapp",
     "private": true,
     "scripts": {

querybook/server/const/metastore.py

@@ -4,3 +4,38 @@
 class DataTableWarningSeverity(Enum):
     WARNING = 0
     ERROR = 1
+
+
+class MetadataType(Enum):
+    TABLE_DESCRIPTION = "table_description"
+    COLUMN_DESCRIPTION = "column_description"
+    OWNER = "owner"
+    TAG = "tag"
+
+
+class MetadataMode(Enum):
+    # Metadata will be read-only on Querybook UI and it will redirect to the metastore link for editing.
+    READ_ONLY = "read_only"
+
+    # On saving, metadata will only be written to querybook db. This is the default mode if not specified.
+    WRITE_LOCAL = "write_local"
+
+    # On saving, metadata will be written back to metastore, as well as querybook db
+    WRITE_BACK = "write_back"
+
+
+class MetastoreLoaderConfig:
+    """Config to set the read/write mode (MetadataMode) for each metadata."""
+
+    _default_config = {
+        MetadataType.TABLE_DESCRIPTION: MetadataMode.WRITE_LOCAL,
+        MetadataType.COLUMN_DESCRIPTION: MetadataMode.WRITE_LOCAL,
+        MetadataType.OWNER: MetadataMode.WRITE_LOCAL,
+        MetadataType.TAG: MetadataMode.WRITE_LOCAL,
+    }
+
+    def __init__(self, config: dict[MetadataType, MetadataMode]):
+        self._config = {**self._default_config, **config}
+
+    def to_dict(self):
+        return {key.value: value.value for (key, value) in self._config.items()}

querybook/server/datasources/metastore.py

@@ -13,7 +13,7 @@
 from app.datasource import register, api_assert, with_impression, admin_only
 from app.flask_app import cache, limiter
 from const.impression import ImpressionItemType
-from const.metastore import DataTableWarningSeverity
+from const.metastore import DataTableWarningSeverity, MetadataType
 from const.time import seconds_in_a_day
 from lib.lineage.utils import lineage
 from lib.metastore.utils import DataTableFinder
@@ -82,6 +82,25 @@ def get_table(table_id, with_schema=True, with_column=True, with_warnings=True):
         return result
 
 
+@register("/table/<int:table_id>/metastore_link/", methods=["GET"])
+def get_table_metastore_link(
+    table_id,
+    metadata_type,
+):
+    verify_data_table_permission(table_id)
+
+    table = logic.get_table_by_id(table_id)
+    schema = table.data_schema
+    metastore_id = schema.metastore_id
+    metastore_loader = get_metastore_loader(metastore_id)
+
+    return metastore_loader.get_metastore_link(
+        metadata_type=MetadataType(metadata_type),
+        schema_name=schema.name,
+        table_name=table.name,
+    )
+
+
 @register("/table_name/<schema_name>/<table_name>/", methods=["GET"])
 def get_table_by_name(
     schema_name,

querybook/server/lib/metastore/base_metastore_loader.py

@@ -5,6 +5,7 @@
 import traceback
 
 from app.db import DBSession, with_session
+from const.metastore import MetadataType, MetastoreLoaderConfig
 from lib.logger import get_logger
 
 from lib.form import AllFormField
@@ -77,10 +78,28 @@ class DataColumn(NamedTuple):
 
 
 class BaseMetastoreLoader(metaclass=ABCMeta):
+    loader_config: MetastoreLoaderConfig = MetastoreLoaderConfig({})
+
     def __init__(self, metastore_dict: Dict):
         self.metastore_id = metastore_dict["id"]
         self.acl_checker = MetastoreTableACLChecker(metastore_dict["acl_control"])
 
+    @classmethod
+    def get_metastore_link(
+        cls, metadata_type: MetadataType, schema_name: str, table_name: str
+    ) -> str:
+        """Return the external metastore link of the table metadata if it has an accessible page for the given type.
+
+        Args:
+            metadata_type (MetadataType): metadata type
+            schema_name (str): table schema name
+            table_name (str): table name
+
+        Returns:
+            str: external metastore link of the table metadata.
+        """
+        return None
+
     @with_session
     def sync_table(
         self, schema_name: str, table_name: str, session=None

querybook/server/models/admin.py

@@ -157,7 +157,14 @@ class QueryMetastore(CRUDMixin, Base):
     acl_control = sql.Column(sql.JSON, default={}, nullable=False)
 
     def to_dict(self):
-        return {"id": self.id, "name": self.name}
+        from lib.metastore import get_metastore_loader_class_by_name
+
+        loader_class = get_metastore_loader_class_by_name(self.loader)
+        return {
+            "id": self.id,
+            "name": self.name,
+            "config": loader_class.loader_config.to_dict(),
+        }
 
     def to_dict_admin(self):
         return {

querybook/webapp/components/DataTableView/DataTableView.tsx

@@ -15,7 +15,11 @@ import { DataTableViewSamples } from 'components/DataTableViewSamples/DataTableV
 import { DataTableViewSourceQuery } from 'components/DataTableViewSourceQuery/DataTableViewSourceQuery';
 import { DataTableViewWarnings } from 'components/DataTableViewWarnings/DataTableViewWarnings';
 import { ComponentType, ElementType } from 'const/analytics';
-import { IPaginatedQuerySampleFilters } from 'const/metastore';
+import {
+    IPaginatedQuerySampleFilters,
+    MetadataMode,
+    MetadataType,
+} from 'const/metastore';
 import { trackClick, trackView } from 'lib/analytics';
 import { setBrowserTitle } from 'lib/querybookUI';
 import history from 'lib/router-history';
@@ -26,6 +30,7 @@ import { getQueryString, replaceQueryString } from 'lib/utils/query-string';
 import * as dataSourcesActions from 'redux/dataSources/action';
 import { fullTableSelector } from 'redux/dataSources/selector';
 import { Dispatch, IStoreState } from 'redux/store/types';
+import { TableResource } from 'resource/table';
 import { Container } from 'ui/Container/Container';
 import { ErrorPage } from 'ui/ErrorPage/ErrorPage';
 import { FourOhFour } from 'ui/ErrorPage/FourOhFour';
@@ -132,6 +137,24 @@ class DataTableViewComponent extends React.PureComponent<
         }));
     }
 
+    @bind
+    public async onEditMetadata(metadataType: MetadataType) {
+        const { table } = this.props;
+        const { data: link } = await TableResource.getMetastoreLink(
+            table.id,
+            metadataType
+        );
+        window.open(link, '_blank');
+    }
+
+    @bind
+    public getOnEditMetadata(metadataType: MetadataType) {
+        const { metastore } = this.props;
+        return metastore.config[metadataType] === MetadataMode.READ_ONLY
+            ? this.onEditMetadata.bind(this, metadataType)
+            : undefined;
+    }
+
     @bind
     public onTabSelected(key) {
         const elementType = tabDefinitions.find(
@@ -168,20 +191,25 @@ class DataTableViewComponent extends React.PureComponent<
                 onTabSelected={this.onTabSelected}
                 updateDataTableDescription={this.updateDataTableDescription}
                 onExampleFilter={this.handleExampleFilter}
+                onEditTableDescriptionRedirect={this.getOnEditMetadata(
+                    MetadataType.TABLE_DESCRIPTION
+                )}
             />
         );
     }
 
     @bind
     public makeColumnsDOM(numberOfRows = null) {
         const { table, tableColumns, updateDataColumnDescription } = this.props;
-
         return (
             <DataTableViewColumn
                 table={table}
                 tableColumns={tableColumns}
                 numberOfRows={numberOfRows}
                 updateDataColumnDescription={updateDataColumnDescription}
+                onEditColumnDescriptionRedirect={this.getOnEditMetadata(
+                    MetadataType.COLUMN_DESCRIPTION
+                )}
             />
         );
     }
@@ -381,7 +409,7 @@ function mapStateToProps(state: IStoreState, ownProps) {
 
     const { tableId } = ownProps;
 
-    const { table, schema, tableName, tableColumns, tableWarnings } =
+    const { table, schema, tableName, tableColumns, tableWarnings, metastore } =
         fullTableSelector(state, tableId);
 
     return {
@@ -395,6 +423,7 @@ function mapStateToProps(state: IStoreState, ownProps) {
         dataJobMetadataById,
         dataSchemasById,
         tableWarnings,
+        metastore,
 
         userInfo: state.user.myUserInfo,
     };

querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.tsx

@@ -4,6 +4,7 @@ import React, { useMemo } from 'react';
 import { DataTableColumnStats } from 'components/DataTableStats/DataTableColumnStats';
 import { IDataColumn } from 'const/metastore';
 import { useToggleState } from 'hooks/useToggleState';
+import { Nullable } from 'lib/typescript';
 import { parseType } from 'lib/utils/complex-types';
 import { Card } from 'ui/Card/Card';
 import { EditableTextField } from 'ui/EditableTextField/EditableTextField';
@@ -17,6 +18,7 @@ import './DataTableColumnCard.scss';
 
 interface IProps {
     column: IDataColumn;
+    onEditColumnDescriptionRedirect?: Nullable<() => Promise<void>>;
     updateDataColumnDescription: (
         columnId: number,
         description: ContentState
@@ -25,6 +27,7 @@ interface IProps {
 
 export const DataTableColumnCard: React.FunctionComponent<IProps> = ({
     column,
+    onEditColumnDescriptionRedirect,
     updateDataColumnDescription,
 }) => {
     const [expanded, , toggleExpanded] = useToggleState(false);
@@ -35,6 +38,7 @@ export const DataTableColumnCard: React.FunctionComponent<IProps> = ({
             value={column.description as ContentState}
             onSave={updateDataColumnDescription.bind(null, column.id)}
             placeholder="No user comments yet for column."
+            onEditRedirect={onEditColumnDescriptionRedirect}
         />
     );
     return (

querybook/webapp/components/DataTableViewColumn/DataTableViewColumn.tsx

@@ -2,6 +2,7 @@ import { ContentState } from 'draft-js';
 import React from 'react';
 
 import { IDataColumn, IDataTable } from 'const/metastore';
+import { Nullable } from 'lib/typescript';
 import { Loading } from 'ui/Loading/Loading';
 import { SearchBar } from 'ui/SearchBar/SearchBar';
 
@@ -15,6 +16,7 @@ export interface IDataTableViewColumnProps {
         columnId: number,
         description: ContentState
     ) => any;
+    onEditColumnDescriptionRedirect?: Nullable<() => Promise<void>>;
 }
 
 export const DataTableViewColumn: React.FunctionComponent<
@@ -24,6 +26,7 @@ export const DataTableViewColumn: React.FunctionComponent<
     table = null,
     tableColumns = [],
     numberOfRows = null,
+    onEditColumnDescriptionRedirect,
 }) => {
     const [filterString, setFilterString] = React.useState('');
 
@@ -60,6 +63,7 @@ export const DataTableViewColumn: React.FunctionComponent<
             column={col}
             updateDataColumnDescription={updateDataColumnDescription}
             key={col.id}
+            onEditColumnDescriptionRedirect={onEditColumnDescriptionRedirect}
         />
     ));
 

querybook/webapp/components/DataTableViewOverview/DataTableViewOverview.tsx

@@ -24,6 +24,7 @@ import {
     IPaginatedQuerySampleFilters,
 } from 'const/metastore';
 import { useMounted } from 'hooks/useMounted';
+import { Nullable } from 'lib/typescript';
 import { titleize } from 'lib/utils';
 import { generateFormattedDate } from 'lib/utils/datetime';
 import { getAppName } from 'lib/utils/global';
@@ -86,6 +87,7 @@ export interface IQuerybookTableViewOverviewProps {
     tableColumns: IDataColumn[];
     tableWarnings: IDataTableWarning[];
 
+    onEditTableDescriptionRedirect?: Nullable<() => Promise<void>>;
     onTabSelected: (key: string) => any;
     updateDataTableDescription: (
         tableId: number,
@@ -102,6 +104,7 @@ export const DataTableViewOverview: React.FC<
     tableWarnings,
     onExampleFilter,
     updateDataTableDescription,
+    onEditTableDescriptionRedirect,
 }) => {
     const onDescriptionSave = useCallback(
         (description: DraftJs.ContentState) =>
@@ -116,6 +119,7 @@ export const DataTableViewOverview: React.FC<
             value={table.description as DraftJs.ContentState}
             onSave={onDescriptionSave}
             placeholder="No description for this table yet."
+            onEditRedirect={onEditTableDescriptionRedirect}
         />
     ) : null;
 

querybook/webapp/const/metastore.ts

@@ -1,8 +1,24 @@
 import type { ContentState } from 'draft-js';
 
+// Keep it in sync with MetadataType in server/const/metastore.py
+export enum MetadataType {
+    TABLE_DESCRIPTION = 'table_description',
+    COLUMN_DESCRIPTION = 'column_description',
+    OWNER = 'owner',
+    TAG = 'tag',
+    DOMAIN = 'domain',
+}
+
+// Keep it in sync with MetadataMode in server/const/metastore.py
+export enum MetadataMode {
+    READ_ONLY = 'read_only',
+    WRITE_LOCAL = 'write_local',
+    WRITE_BACK = 'write_back',
+}
 export interface IQueryMetastore {
     id: number;
     name: string;
+    config: Record<MetadataType, MetadataMode>;
 }
 
 export interface IDataSchema {

querybook/webapp/redux/dataSources/selector.ts

@@ -34,11 +34,13 @@ export const fullTableSelector = createSelector(
     (state: IStoreState) => state.dataSources.dataSchemasById,
     (state: IStoreState) => state.dataSources.dataColumnsById,
     (state: IStoreState) => state.dataSources.dataTableWarningById,
+    (state: IStoreState) => state.dataSources.queryMetastoreById,
     (
         tableFromState,
         dataSchemasById,
         dataColumnsById,
-        dataTableWarningById
+        dataTableWarningById,
+        queryMetastoreById
     ) => {
         const schemaFromState = tableFromState
             ? dataSchemasById[tableFromState.schema]
@@ -64,6 +66,7 @@ export const fullTableSelector = createSelector(
             tableWarnings: (tableFromState?.warnings ?? [])
                 .map((id) => dataTableWarningById[id])
                 .filter((warning) => warning),
+            metastore: queryMetastoreById[schemaFromState.metastore_id],
         };
     }
 );

querybook/webapp/resource/table.ts

@@ -20,6 +20,7 @@ import type {
     ITopQueryConcurrences,
     ITopQueryUser,
     IUpdateTableParams,
+    MetadataType,
 } from 'const/metastore';
 import type { ITag } from 'const/tag';
 import ds from 'lib/datasource';
@@ -131,6 +132,11 @@ export const TableResource = {
             metastore_id: metastoreId,
         }),
 
+    getMetastoreLink: (tableId: number, metadataType: MetadataType) =>
+        ds.fetch<string>(`/table/${tableId}/metastore_link/`, {
+            metadata_type: metadataType,
+        }),
+
     checkIfExists: (
         metastoreId: number,
         schemaName: string,