present column lineage of a dataset (#2293)

* display column lineage JSON Signed-off-by: Pawel Leszczynski <[email protected]> * refactoring for display column lineage JSON (#2294) Signed-off-by: tito1212 <[email protected]> Signed-off-by: tito1212 <[email protected]> Co-authored-by: tito1212 <[email protected]> * Refactoring for display column lineage JSON (#2301) * refactoring for display column lineage JSON Signed-off-by: tito1212 <[email protected]> * request api after click Signed-off-by: tito1212 <[email protected]> Signed-off-by: tito1212 <[email protected]> Co-authored-by: tito1212 <[email protected]> * call backend on tab switch Signed-off-by: Pawel Leszczynski <[email protected]> * changes after review. refactoring. fix type and name of file Signed-off-by: tito12 <[email protected]> * Changes after review. Separate fetching data in child component Signed-off-by: tito12 <[email protected]> * eslint fix changes Signed-off-by: tito12 <[email protected]> Signed-off-by: Pawel Leszczynski <[email protected]> Signed-off-by: tito1212 <[email protected]> Signed-off-by: tito12 <[email protected]> Co-authored-by: tito12 <[email protected]> Co-authored-by: tito1212 <[email protected]> Co-authored-by: Pawel Leszczynski <[email protected]>
MarquezProject · Dec 27, 2022 · 39bcd3a · 39bcd3a
1 parent 65669ca
commit 39bcd3a
Show file tree

Hide file tree

Showing 13 changed files with 230 additions and 33 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,11 @@
 
 ### Added
 
+* Column-lineage endpoints supports point-in-time requests [`#2265`](https://github.com/MarquezProject/marquez/pull/2265) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
+    *Enable requesting `column-lineage` endpoint by a dataset version, job version or dataset field of a specific dataset version.*
+* Present column lineage of a dataset [`#2293`](https://github.com/MarquezProject/marquez/pull/2293) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
+    *Column lineage of a dataset with a single level of depth can
+    be displayed in datase details tab.*
 * Add point-in-time requests support to column-lineage endpoints [`#2265`](https://github.com/MarquezProject/marquez/pull/2265) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)  
     *Enables requesting `column-lineage` endpoint by a dataset version, job version or dataset field of a specific dataset version.*
 * Add column lineage point-in-time Java client methods [`#2269`](https://github.com/MarquezProject/marquez/pull/2269) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)  

diff --git a/web/src/components/bottom-bar/BottomBar.tsx b/web/src/components/bottom-bar/BottomBar.tsx
@@ -54,7 +54,7 @@ class BottomBar extends React.Component<BottomBarProps> {
         <Box className={classes.overflow} height={bottomBarHeight}>
           <Container maxWidth={'lg'} disableGutters={true}>
             {lineageJob && <JobDetailPage job={lineageJob} />}
-            {lineageDataset && <DatasetDetailPage dataset={lineageDataset} />}
+            {lineageDataset && <DatasetDetailPage lineageDataset={lineageDataset} />}
           </Container>
         </Box>
       </Box>

diff --git a/web/src/components/datasets/DatasetColumnLineage.tsx b/web/src/components/datasets/DatasetColumnLineage.tsx
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import * as Redux from 'redux'
+import { Box, Button } from '@material-ui/core'
+import { Dataset } from '../../types/api'
+import { IState } from '../../store/reducers'
+import { LineageDataset } from '../lineage/types'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+import { fetchDataset, resetDataset } from '../../store/actionCreators'
+import { fileSize } from '../../helpers'
+import { saveAs } from 'file-saver'
+import MqEmpty from '../core/empty/MqEmpty'
+import MqJson from '../core/code/MqJson'
+import MqText from '../core/text/MqText'
+import React, { FunctionComponent, useEffect } from 'react'
+
+interface DatasetColumnLineageProps {
+  lineageDataset: LineageDataset
+}
+
+interface StateProps {
+  dataset: Dataset
+}
+
+interface DispatchProps {
+  fetchDataset: typeof fetchDataset
+  resetDataset: typeof resetDataset
+}
+
+type IProps = DatasetColumnLineageProps & DispatchProps & StateProps
+
+const DatasetColumnLineage: FunctionComponent<IProps> = props => {
+  const { dataset, lineageDataset, fetchDataset, resetDataset } = props
+  const columnLineage = dataset.columnLineage
+
+  useEffect(() => {
+    fetchDataset(lineageDataset.namespace, lineageDataset.name)
+  }, [lineageDataset.name])
+
+  // unmounting
+  useEffect(
+    () => () => {
+      resetDataset()
+    },
+    []
+  )
+
+  const handleDownloadPayload = (data: object) => {
+    const title = `${lineageDataset.name}-${lineageDataset.namespace}-columnLineage`
+    const blob = new Blob([JSON.stringify(data)], { type: 'application/json' })
+    saveAs(blob, `${title}.json`)
+  }
+
+  return (
+    <>
+      {columnLineage ? (
+        <>
+          {fileSize(JSON.stringify(columnLineage)).kiloBytes > 500 ? (
+            <Box p={2}>
+              <MqEmpty title={'Payload is too big for render'}>
+                <div>
+                  <MqText subdued>Please click on button and download payload as file</MqText>
+                  <br />
+                  <Button
+                    variant='outlined'
+                    color='primary'
+                    onClick={() => handleDownloadPayload(columnLineage)}
+                  >
+                    Download payload
+                  </Button>
+                </div>
+              </MqEmpty>
+            </Box>
+          ) : (
+            <MqJson code={columnLineage} wrapLongLines={true} showLineNumbers={true} />
+          )}
+        </>
+      ) : (
+        <MqEmpty
+          title={'No column lineage'}
+          body={'Column lineage not available for the specified dataset.'}
+        />
+      )}
+    </>
+  )
+}
+
+const mapStateToProps = (state: IState) => ({
+  dataset: state.dataset.result
+})
+
+const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
+  bindActionCreators(
+    {
+      fetchDataset: fetchDataset,
+      resetDataset: resetDataset
+    },
+    dispatch
+  )
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(DatasetColumnLineage)
diff --git a/web/src/components/datasets/DatasetDetailPage.tsx b/web/src/components/datasets/DatasetDetailPage.tsx
@@ -1,7 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
 
-import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
-
 import * as Redux from 'redux'
 import { Box, Chip, Tab, Tabs } from '@material-ui/core'
 import { DatasetVersion } from '../../types/api'
@@ -15,14 +13,20 @@ import {
 import { LineageDataset } from '../lineage/types'
 import { bindActionCreators } from 'redux'
 import { connect } from 'react-redux'
-import { fetchDatasetVersions, resetDatasetVersions } from '../../store/actionCreators'
+import {
+  fetchDatasetVersions,
+  resetDataset,
+  resetDatasetVersions
+} from '../../store/actionCreators'
 import { useHistory } from 'react-router-dom'
 import CircularProgress from '@material-ui/core/CircularProgress/CircularProgress'
 import CloseIcon from '@material-ui/icons/Close'
+import DatasetColumnLineage from './DatasetColumnLineage'
 import DatasetInfo from './DatasetInfo'
 import DatasetVersions from './DatasetVersions'
 import IconButton from '@material-ui/core/IconButton'
 import MqText from '../core/text/MqText'
+import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
 
 const styles = ({ spacing }: ITheme) => {
   return createStyles({
@@ -40,29 +44,20 @@ const styles = ({ spacing }: ITheme) => {
       '&:not(:last-of-type)': {
         marginRight: spacing(1)
       }
-    },
-    noData: {
-      padding: '125px 0 0 0'
-    },
-    infoIcon: {
-      paddingLeft: '3px',
-      paddingTop: '3px'
-    },
-    updated: {
-      marginTop: '10px'
     }
   })
 }
 
 interface StateProps {
-  dataset: LineageDataset
+  lineageDataset: LineageDataset
   versions: DatasetVersion[]
   versionsLoading: boolean
 }
 
 interface DispatchProps {
   fetchDatasetVersions: typeof fetchDatasetVersions
   resetDatasetVersions: typeof resetDatasetVersions
+  resetDataset: typeof resetDataset
 }
 
 type IProps = IWithStyles<typeof styles> & StateProps & DispatchProps
@@ -75,21 +70,30 @@ function a11yProps(index: number) {
 }
 
 const DatasetDetailPage: FunctionComponent<IProps> = props => {
-  const { classes, fetchDatasetVersions, resetDatasetVersions, versions, versionsLoading } = props
+  const {
+    classes,
+    fetchDatasetVersions,
+    resetDataset,
+    resetDatasetVersions,
+    versions,
+    versionsLoading
+  } = props
   const { root } = classes
   const history = useHistory()
   const i18next = require('i18next')
 
   useEffect(() => {
-    fetchDatasetVersions(props.dataset.namespace, props.dataset.name)
-  }, [props.dataset.name])
+    fetchDatasetVersions(props.lineageDataset.namespace, props.lineageDataset.name)
+  }, [props.lineageDataset.name])
 
   // unmounting
-  useEffect(() => {
-    return () => {
+  useEffect(
+    () => () => {
+      resetDataset()
       resetDatasetVersions()
-    }
-  }, [])
+    },
+    []
+  )
 
   const [tab, setTab] = React.useState(0)
   const handleChange = (event: ChangeEvent, newValue: SetStateAction<number>) => {
@@ -108,8 +112,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
     return null
   }
 
-  const dataset = versions[0]
-  const { name, tags, description } = dataset
+  const firstVersion = versions[0]
+  const { name, tags, description } = firstVersion
 
   return (
     <Box my={2} className={root}>
@@ -136,6 +140,11 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
                 {...a11yProps(1)}
                 disableRipple={true}
               />
+              <Tab
+                label={i18next.t('datasets.column_lineage')}
+                {...a11yProps(1)}
+                disableRipple={true}
+              />
             </Tabs>
           </Box>
           <IconButton onClick={() => history.push('/datasets')}>
@@ -151,12 +160,13 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
       </Box>
       {tab === 0 && (
         <DatasetInfo
-          datasetFields={dataset.fields}
-          facets={dataset.facets}
-          run={dataset.createdByRun}
+          datasetFields={firstVersion.fields}
+          facets={firstVersion.facets}
+          run={firstVersion.createdByRun}
         />
       )}
       {tab === 1 && <DatasetVersions versions={props.versions} />}
+      {tab === 2 && <DatasetColumnLineage lineageDataset={props.lineageDataset} />}
     </Box>
   )
 }
@@ -171,7 +181,8 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
   bindActionCreators(
     {
       fetchDatasetVersions: fetchDatasetVersions,
-      resetDatasetVersions: resetDatasetVersions
+      resetDatasetVersions: resetDatasetVersions,
+      resetDataset: resetDataset
     },
     dispatch
   )

diff --git a/web/src/components/sidenav/Sidenav.tsx b/web/src/components/sidenav/Sidenav.tsx
@@ -90,9 +90,9 @@ class Sidenav extends React.Component<SidenavProps> {
                 title={i18next.t('sidenav.events')}
                 active={this.props.location.pathname === '/events'}
               >
-                <SVG 
-                  src="https://raw.githubusercontent.com/MarquezProject/marquez/main/web/src/img/iconSearchArrow.svg" 
-                  width={'30px'} 
+                <SVG
+                  src='https://raw.githubusercontent.com/MarquezProject/marquez/main/web/src/img/iconSearchArrow.svg'
+                  width={'30px'}
                 />
               </MqIconButton>
             </RouterLink>

diff --git a/web/src/i18n/config.ts b/web/src/i18n/config.ts
@@ -58,7 +58,8 @@ i18next
           },
           datasets: {
             latest_tab: 'LATEST SCHEMA',
-            history_tab: 'VERSION HISTORY'
+            history_tab: 'VERSION HISTORY',
+            column_lineage: 'COLUMN LINEAGE'
           },
           datasets_route: {
             empty_title: 'No datasets found',

diff --git a/web/src/store/actionCreators/actionTypes.ts b/web/src/store/actionCreators/actionTypes.ts
@@ -22,8 +22,11 @@ export const RESET_JOBS = 'RESET_JOBS'
 
 // datasets
 export const FETCH_DATASETS = 'FETCH_DATASETS'
+export const FETCH_DATASET = 'FETCH_DATASET'
 export const FETCH_DATASETS_SUCCESS = 'FETCH_DATASETS_SUCCESS'
+export const FETCH_DATASET_SUCCESS = 'FETCH_DATASET_SUCCESS'
 export const RESET_DATASETS = 'RESET_DATASETS'
+export const RESET_DATASET = 'RESET_DATASET'
 export const FETCH_DATASET_VERSIONS = 'FETCH_DATASET_VERSIONS'
 export const FETCH_DATASET_VERSIONS_SUCCESS = 'FETCH_DATASET_VERSIONS_SUCCESS'
 export const RESET_DATASET_VERSIONS = 'RESET_DATASET_VERSIONS'

diff --git a/web/src/store/actionCreators/index.ts b/web/src/store/actionCreators/index.ts
@@ -48,6 +48,21 @@ export const fetchDatasetsSuccess = (datasets: Dataset[]) => ({
   }
 })
 
+export const fetchDataset = (namespace: string, name: string) => ({
+  type: actionTypes.FETCH_DATASET,
+  payload: {
+    namespace,
+    name
+  }
+})
+
+export const fetchDatasetSuccess = (dataset: Dataset) => ({
+  type: actionTypes.FETCH_DATASET_SUCCESS,
+  payload: {
+    dataset
+  }
+})
+
 export const fetchDatasetVersions = (namespace: string, name: string) => ({
   type: actionTypes.FETCH_DATASET_VERSIONS,
   payload: {
@@ -67,6 +82,10 @@ export const resetDatasetVersions = () => ({
   type: actionTypes.RESET_DATASET_VERSIONS
 })
 
+export const resetDataset = () => ({
+  type: actionTypes.RESET_DATASET
+})
+
 export const resetDatasets = () => ({
   type: actionTypes.RESET_DATASETS
 })

diff --git a/web/src/store/reducers/dataset.ts b/web/src/store/reducers/dataset.ts
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { Dataset } from '../../types/api'
+import { FETCH_DATASET, FETCH_DATASET_SUCCESS, RESET_DATASET } from '../actionCreators/actionTypes'
+import { fetchDatasetSuccess } from '../actionCreators'
+
+export type IDatasetState = { isLoading: boolean; result: Dataset; init: boolean }
+
+export const initialState: IDatasetState = { isLoading: false, init: false, result: {} as Dataset }
+
+type IDatasetAction = ReturnType<typeof fetchDatasetSuccess>
+
+export default (state: IDatasetState = initialState, action: IDatasetAction): IDatasetState => {
+  const { type, payload } = action
+
+  switch (type) {
+    case FETCH_DATASET:
+      return { ...state, isLoading: true }
+    case FETCH_DATASET_SUCCESS:
+      return { ...state, isLoading: false, init: true, result: payload.dataset }
+    case RESET_DATASET:
+      return initialState
+    default:
+      return state
+  }
+}
diff --git a/web/src/store/reducers/index.ts b/web/src/store/reducers/index.ts
@@ -3,6 +3,7 @@
 import { History } from 'history'
 import { Reducer, combineReducers } from 'redux'
 import { connectRouter } from 'connected-react-router'
+import dataset, { IDatasetState } from './dataset'
 import datasetVersions, { IDatasetVersionsState } from './datasetVersions'
 import datasets, { IDatasetsState } from './datasets'
 import display, { IDisplayState } from './display'
@@ -15,6 +16,7 @@ import search, { ISearchState } from './search'
 
 export interface IState {
   datasets: IDatasetsState
+  dataset: IDatasetState
   datasetVersions: IDatasetVersionsState
   events: IEventsState
   jobs: IJobsState
@@ -29,6 +31,7 @@ export interface IState {
 export default (history: History): Reducer =>
   combineReducers({
     router: connectRouter(history),
+    dataset,
     datasets,
     datasetVersions,
     events,