UI: Selection on graph with edges. Status for jobs/datasets (#2384)

* Selection on graph with edges. Status for jobs/datasets Signed-off-by: tito12 <[email protected]> --------- Signed-off-by: tito12 <[email protected]> Co-authored-by: Willy Lulciuc <[email protected]>
MarquezProject · Feb 9, 2023 · f9465bf · f9465bf
1 parent 9e0c84b
commit f9465bf
Show file tree

Hide file tree

Showing 20 changed files with 329 additions and 163 deletions.
diff --git a/docker/metadata.json b/docker/metadata.json
@@ -1358,6 +1358,22 @@
             "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json",
             "name": "food_delivery_db",
             "uri": "postgres://food_delivery:food_delivery@postgres:5432/food_delivery"
+          },
+          "dataQualityAssertions": {
+            "_producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json",
+            "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json",
+            "assertions": [
+              {
+                "assertion": "not_null",
+                "success": false,
+                "column": "driver_id"
+              },
+              {
+                "assertion": "is_string",
+                "success": true,
+                "column": "customer_address"
+              }
+            ]
           }
         }
       }
@@ -1824,4 +1840,4 @@
     },
     "producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json"
   }
-]
+]
diff --git a/web/src/components/core/status/MqStatus.tsx b/web/src/components/core/status/MqStatus.tsx
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { theme } from '../../../helpers/theme'
+import Box from '@material-ui/core/Box'
+import MqText from '../text/MqText'
+import React from 'react'
+import createStyles from '@material-ui/core/styles/createStyles'
+import withStyles, { WithStyles } from '@material-ui/core/styles/withStyles'
+
+const styles = () =>
+  createStyles({
+    type: {
+      display: 'flex',
+      alignItems: 'center',
+      gap: theme.spacing(1)
+    },
+    status: {
+      width: theme.spacing(2),
+      height: theme.spacing(2),
+      borderRadius: '50%'
+    }
+  })
+
+interface OwnProps {
+  color: string | null
+  label?: string
+}
+
+const MqStatus: React.FC<OwnProps & WithStyles<typeof styles>> = ({ label, color, classes }) => {
+  if (!color) {
+    return null
+  }
+  return (
+    <Box className={classes.type}>
+      <Box className={classes.status} style={{ backgroundColor: color }} />
+      {label && <MqText>{label}</MqText>}
+    </Box>
+  )
+}
+
+export default withStyles(styles)(MqStatus)
diff --git a/web/src/components/datasets/DatasetColumnLineage.tsx b/web/src/components/datasets/DatasetColumnLineage.tsx
@@ -102,7 +102,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(DatasetColumnLineage)
+export default connect(mapStateToProps, mapDispatchToProps)(DatasetColumnLineage)
diff --git a/web/src/components/datasets/DatasetDetailPage.tsx b/web/src/components/datasets/DatasetDetailPage.tsx
@@ -15,6 +15,7 @@ import { LineageDataset } from '../lineage/types'
 import { alpha } from '@material-ui/core/styles'
 import { bindActionCreators } from 'redux'
 import { connect } from 'react-redux'
+import { datasetFacetsStatus } from '../../helpers/nodes'
 import {
   deleteDataset,
   dialogToggle,
@@ -31,7 +32,9 @@ import DatasetInfo from './DatasetInfo'
 import DatasetVersions from './DatasetVersions'
 import Dialog from '../Dialog'
 import IconButton from '@material-ui/core/IconButton'
+import MqStatus from '../core/status/MqStatus'
 import MqText from '../core/text/MqText'
+
 import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
 
 const styles = ({ spacing }: ITheme) => {
@@ -143,6 +146,7 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
 
   const firstVersion = versions[0]
   const { name, tags, description } = firstVersion
+  const facetsStatus = datasetFacetsStatus(firstVersion.facets)
 
   return (
     <Box my={2} className={root}>
@@ -202,9 +206,16 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
             </IconButton>
           </Box>
         </Box>
-        <MqText heading font={'mono'}>
-          {name}
-        </MqText>
+        <Box display={'flex'} alignItems={'center'}>
+          {facetsStatus && (
+            <Box mr={1}>
+              <MqStatus color={facetsStatus} />
+            </Box>
+          )}
+          <MqText heading font={'mono'}>
+            {name}
+          </MqText>
+        </Box>
         <Box mb={2}>
           <MqText subdued>{description}</MqText>
         </Box>
@@ -241,7 +252,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(withStyles(styles)(DatasetDetailPage))
+export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(DatasetDetailPage))
diff --git a/web/src/components/datasets/DatasetVersions.tsx b/web/src/components/datasets/DatasetVersions.tsx
@@ -31,9 +31,8 @@ interface DatasetVersionsProps {
   versions: DatasetVersion[]
 }
 
-const DatasetVersions: FunctionComponent<
-  DatasetVersionsProps & IWithStyles<typeof styles>
-> = props => {
+const DatasetVersions: FunctionComponent<DatasetVersionsProps &
+  IWithStyles<typeof styles>> = props => {
   const { versions, classes } = props
 
   const [infoView, setInfoView] = React.useState<DatasetVersion | null>(null)

diff --git a/web/src/components/jobs/JobDetailPage.tsx b/web/src/components/jobs/JobDetailPage.tsx
@@ -25,15 +25,16 @@ import {
   resetJobs,
   resetRuns
 } from '../../store/actionCreators'
+import { jobRunsStatus } from '../../helpers/nodes'
 import { theme } from '../../helpers/theme'
 import { useHistory } from 'react-router-dom'
 import CloseIcon from '@material-ui/icons/Close'
 import Dialog from '../Dialog'
 import IconButton from '@material-ui/core/IconButton'
 import MqEmpty from '../core/empty/MqEmpty'
+import MqStatus from '../core/status/MqStatus'
 import MqText from '../core/text/MqText'
 import RunInfo from './RunInfo'
-import RunStatus from './RunStatus'
 import Runs from './Runs'
 
 const styles = ({ spacing }: ITheme) => {
@@ -166,9 +167,9 @@ const JobDetailPage: FunctionComponent<IProps> = props => {
         </Box>
       </Box>
       <Box display={'flex'} alignItems={'center'}>
-        {job.latestRun && (
+        {runs.length && (
           <Box mr={1}>
-            <RunStatus run={job.latestRun} />
+            <MqStatus color={jobRunsStatus(runs)} />
           </Box>
         )}
         <MqText font={'mono'} heading>
@@ -212,7 +213,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(withStyles(styles)(JobDetailPage))
+export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(JobDetailPage))
diff --git a/web/src/components/jobs/RunStatus.tsx b/web/src/components/jobs/RunStatus.tsx
@@ -3,8 +3,7 @@
 
 import { Box, Theme, Tooltip, WithStyles, createStyles, withStyles } from '@material-ui/core'
 import { Run } from '../../types/api'
-
-import { runColorMap } from '../../helpers/runs'
+import { runStateColor } from '../../helpers/nodes'
 
 import React, { FunctionComponent } from 'react'
 
@@ -26,7 +25,11 @@ const RunStatus: FunctionComponent<RunStatusProps & WithStyles<typeof styles>> =
   const { run, classes } = props
   return (
     <Tooltip title={run.state}>
-      <Box mr={1} className={classes.status} style={{ backgroundColor: runColorMap[run.state] }} />
+      <Box
+        mr={1}
+        className={classes.status}
+        style={{ backgroundColor: runStateColor(run.state) }}
+      />
     </Tooltip>
   )
 }

diff --git a/web/src/components/lineage/Lineage.tsx b/web/src/components/lineage/Lineage.tsx
@@ -85,6 +85,7 @@ class Lineage extends React.Component<LineageProps, LineageState> {
     const nodeName = this.props.match.params.nodeName
     const namespace = this.props.match.params.namespace
     const nodeType = this.props.match.params.nodeType
+
     if (nodeName && namespace && nodeType) {
       const nodeId = generateNodeId(
         this.props.match.params.nodeType.toUpperCase() as JobOrDataset,
@@ -114,6 +115,7 @@ class Lineage extends React.Component<LineageProps, LineageState> {
         this.props.match.params.namespace,
         this.props.match.params.nodeName
       )
+      this.getEdges()
     }
   }
 
@@ -129,6 +131,48 @@ class Lineage extends React.Component<LineageProps, LineageState> {
     })
   }
 
+  getEdges = () => {
+    const selectedPaths = this.getSelectedPaths()
+
+    return g?.edges().map(e => {
+      const isSelected = selectedPaths.some((r: any) => e.v === r[0] && e.w === r[1])
+      return Object.assign(g.edge(e), { isSelected: isSelected })
+    })
+  }
+
+  getSelectedPaths = () => {
+    const paths = [] as Array<[string, string]>
+
+    const getSuccessors = (node: string) => {
+      const successors = g?.successors(node)
+      if (successors?.length) {
+        for (let i = 0; i < node.length - 1; i++) {
+          if (successors[i]) {
+            paths.push([node, (successors[i] as unknown) as string])
+            getSuccessors((successors[i] as unknown) as string)
+          }
+        }
+      }
+    }
+
+    const getPredecessors = (node: string) => {
+      const predecessors = g?.predecessors(node)
+      if (predecessors?.length) {
+        for (let i = 0; i < node.length - 1; i++) {
+          if (predecessors[i]) {
+            paths.push([(predecessors[i] as unknown) as string, node])
+            getPredecessors((predecessors[i] as unknown) as string)
+          }
+        }
+      }
+    }
+
+    getSuccessors(this.props.selectedNode)
+    getPredecessors(this.props.selectedNode)
+
+    return paths
+  }
+
   buildGraphAll = (graph: LineageNode[]) => {
     // nodes
     for (let i = 0; i < graph.length; i++) {
@@ -150,14 +194,15 @@ class Lineage extends React.Component<LineageProps, LineageState> {
 
     this.setState({
       graph: g,
-      edges: g.edges().map(e => g.edge(e)),
+      edges: this.getEdges(),
       nodes: g.nodes().map(v => g.node(v))
     })
   }
 
   render() {
     const { classes } = this.props
     const i18next = require('i18next')
+
     return (
       <Box className={classes.lineageContainer}>
         {this.props.selectedNode === null && (
@@ -227,9 +272,6 @@ class Lineage extends React.Component<LineageProps, LineageState> {
                             <Node
                               key={node.data.name}
                               node={node}
-                              edgeEnds={this.state.edges.map(
-                                edge => edge.points[edge.points.length - 1]
-                              )}
                               selectedNode={this.props.selectedNode}
                             />
                           ))}
@@ -262,9 +304,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default withStyles(styles)(
-  connect(
-    mapStateToProps,
-    mapDispatchToProps
-  )(withRouter(Lineage))
-)
+export default withStyles(styles)(connect(mapStateToProps, mapDispatchToProps)(withRouter(Lineage)))
diff --git a/web/src/components/lineage/components/drag-bar/DragBar.tsx b/web/src/components/lineage/components/drag-bar/DragBar.tsx
@@ -107,7 +107,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     },
     dispatch
   )
-export default connect(
-  null,
-  mapDispatchToProps
-)(withStyles(styles)(DragBar))
+export default connect(null, mapDispatchToProps)(withStyles(styles)(DragBar))
diff --git a/web/src/components/lineage/components/edge/Edge.tsx b/web/src/components/lineage/components/edge/Edge.tsx
@@ -1,19 +1,48 @@
 // Copyright 2018-2023 contributors to the Marquez project
 // SPDX-License-Identifier: Apache-2.0
 
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
 import { GraphEdge } from 'dagre'
 import { LinePath } from '@visx/shape'
 import { curveMonotoneX } from '@visx/curve'
+import { faCaretRight } from '@fortawesome/free-solid-svg-icons/faCaretRight'
 import { theme } from '../../../../helpers/theme'
 import React from 'react'
 
 type EdgeProps = {
   edgePoints: GraphEdge[]
 }
 
+type EdgePoint = {
+  isSelected: boolean
+  points: {
+    x: number
+    y: number
+  }[]
+}
+
+const RADIUS = 14
+const OUTER_RADIUS = RADIUS + 8
+const ICON_SIZE = 16
+
 class Edge extends React.Component<EdgeProps> {
+  getPoints = (edge: EdgePoint) => edge.points[edge.points.length - 1]
+
   render() {
     const { edgePoints } = this.props
+    const edgeEnds = edgePoints.map(edge => {
+      const isSelected = edgePoints.find(
+        o =>
+          this.getPoints(o as EdgePoint).x == this.getPoints(edge as EdgePoint).x &&
+          this.getPoints(o as EdgePoint).y == this.getPoints(edge as EdgePoint).y &&
+          o.isSelected === true
+      )
+      return {
+        ...edge.points[edge.points.length - 1],
+        ...{ isSelected: typeof isSelected !== 'undefined' }
+      }
+    })
+
     return (
       <>
         {edgePoints.map((edge, i) => (
@@ -23,12 +52,23 @@ class Edge extends React.Component<EdgeProps> {
             data={edge.points}
             x={(d, index) => (index === 0 ? d.x + 20 : d.x - 25)}
             y={d => d.y}
-            stroke={theme.palette.secondary.main}
+            stroke={edge.isSelected ? theme.palette.common.white : theme.palette.secondary.main}
             strokeWidth={1}
             opacity={1}
             shapeRendering='geometricPrecision'
           />
         ))}
+        {edgeEnds.map((edge, i) => (
+          <FontAwesomeIcon
+            key={i}
+            icon={faCaretRight}
+            x={edge.x - OUTER_RADIUS - ICON_SIZE / 2}
+            y={edge.y - ICON_SIZE / 2}
+            width={ICON_SIZE}
+            height={ICON_SIZE}
+            color={edge.isSelected ? theme.palette.common.white : theme.palette.secondary.main}
+          />
+        ))}
       </>
     )
   }