From dd031699819b96b3f99ab7ae3c32bb91540ceb3d Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 22 Nov 2021 16:07:41 -0600 Subject: [PATCH 01/22] [dagit] Improvements to AssetGraph performance, "Launch Run" functionality --- .../dagit/packages/core/src/graph/OpLinks.tsx | 14 +- .../dagit/packages/core/src/graph/OpNode.tsx | 1 + .../packages/core/src/graph/PipelineGraph.tsx | 63 +-- .../packages/core/src/graph/SVGViewport.tsx | 19 +- .../core/src/pipelines/GraphExplorer.tsx | 12 +- .../src/pipelines/PipelineExplorerRoot.tsx | 2 +- .../asset-graph/AssetGraphExplorer.tsx | 399 +++++++++++------- .../src/workspace/asset-graph/AssetLinks.tsx | 39 ++ .../src/workspace/asset-graph/AssetNode.tsx | 344 ++++++++------- .../src/workspace/asset-graph/ForeignNode.tsx | 5 +- .../asset-graph/SidebarAssetInfo.tsx | 30 +- .../core/src/workspace/asset-graph/Utils.tsx | 31 +- .../asset-graph/types/AssetGraphQuery.ts | 38 +- .../asset-graph/types/AssetNodeFragment.ts | 168 ++++++++ 14 files changed, 770 insertions(+), 395 deletions(-) create mode 100644 js_modules/dagit/packages/core/src/workspace/asset-graph/AssetLinks.tsx create mode 100644 js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts diff --git a/js_modules/dagit/packages/core/src/graph/OpLinks.tsx b/js_modules/dagit/packages/core/src/graph/OpLinks.tsx index 548682ef42913..89cc6b657a280 100644 --- a/js_modules/dagit/packages/core/src/graph/OpLinks.tsx +++ b/js_modules/dagit/packages/core/src/graph/OpLinks.tsx @@ -66,13 +66,13 @@ const inputIsDynamicCollect = ( export const OpLinks = React.memo( (props: { - opacity: number; + color: string; ops: PipelineGraphOpFragment[]; layout: IFullPipelineLayout; connections: ILayoutConnection[]; onHighlight: (arr: Edge[]) => void; }) => ( - + {buildSVGPaths(props.connections, props.layout.ops).map( ({path, from, sourceOutput, targetInput, to}, idx) => ( props.onHighlight([])} onMouseEnter={() => props.onHighlight([{a: from.opName, b: to.opName}])} > - + {outputIsDynamic(props.ops, from) && ( = ({x, y, direction}) => ( + color: string; +}> = ({x, y, direction, color}) => ( ` opacity: ${({$dim}) => ($dim ? 0.3 : 1)}; + pointer-events: auto; .highlight-box { border: ${(p) => diff --git a/js_modules/dagit/packages/core/src/graph/PipelineGraph.tsx b/js_modules/dagit/packages/core/src/graph/PipelineGraph.tsx index 3e02d0720fc71..8d3e496a29435 100644 --- a/js_modules/dagit/packages/core/src/graph/PipelineGraph.tsx +++ b/js_modules/dagit/packages/core/src/graph/PipelineGraph.tsx @@ -37,6 +37,7 @@ interface IPipelineGraphProps { interface IPipelineContentsProps extends IPipelineGraphProps { minified: boolean; layout: IFullPipelineLayout; + bounds: {top: number; left: number; right: number; bottom: number}; } interface IPipelineContentsState { @@ -101,6 +102,7 @@ class PipelineGraphContents extends React.PureComponent< layout, minified, ops, + bounds, focusOps, parentOp, parentHandleID, @@ -151,14 +153,14 @@ class PipelineGraphContents extends React.PureComponent< isHighlighted(this.state.highlighted, { @@ -176,28 +178,38 @@ class PipelineGraphContents extends React.PureComponent< strokeWidth={2} /> ))} - - {ops.map((op) => ( - onClickOp({name: op.name})} - onDoubleClick={() => onDoubleClickOp({name: op.name})} - onEnterComposite={() => onEnterSubgraph({name: op.name})} - onHighlightEdges={this.onHighlightEdges} - layout={layout.ops[op.name]} - selected={selectedOp === op} - focused={focusOps.includes(op)} - highlightedEdges={ - isOpHighlighted(this.state.highlighted, op.name) - ? this.state.highlighted - : EmptyHighlightedArray - } - dim={highlightedOps.length > 0 && highlightedOps.indexOf(op) === -1} - /> - ))} + + {ops + .filter((op) => { + const box = layout.ops[op.name].boundingBox; + return ( + box.x + box.width >= bounds.left && + box.y + box.height >= bounds.top && + box.x < bounds.right && + box.y < bounds.bottom + ); + }) + .map((op) => ( + onClickOp({name: op.name})} + onDoubleClick={() => onDoubleClickOp({name: op.name})} + onEnterComposite={() => onEnterSubgraph({name: op.name})} + onHighlightEdges={this.onHighlightEdges} + layout={layout.ops[op.name]} + selected={selectedOp === op} + focused={focusOps.includes(op)} + highlightedEdges={ + isOpHighlighted(this.state.highlighted, op.name) + ? this.state.highlighted + : EmptyHighlightedArray + } + dim={highlightedOps.length > 0 && highlightedOps.indexOf(op) === -1} + /> + ))} ); @@ -340,7 +352,7 @@ export class PipelineGraph extends React.Component { onClick={onClickBackground} onDoubleClick={this.unfocus} > - {({scale}: any) => ( + {({scale}, bounds) => ( <> { layout={layout} minified={scale < DETAIL_ZOOM - 0.01} onDoubleClickOp={onDoubleClickOp || this.focusOnOp} + bounds={bounds} /> diff --git a/js_modules/dagit/packages/core/src/graph/SVGViewport.tsx b/js_modules/dagit/packages/core/src/graph/SVGViewport.tsx index 691d388b5294e..2f88a52a240c4 100644 --- a/js_modules/dagit/packages/core/src/graph/SVGViewport.tsx +++ b/js_modules/dagit/packages/core/src/graph/SVGViewport.tsx @@ -23,7 +23,10 @@ interface SVGViewportProps { onClick?: (event: React.MouseEvent) => void; onDoubleClick?: (event: React.MouseEvent) => void; onKeyDown: (event: React.KeyboardEvent) => void; - children: (state: SVGViewportState) => React.ReactNode; + children: ( + state: SVGViewportState, + bounds: {top: number; left: number; bottom: number; right: number}, + ) => React.ReactNode; } interface SVGViewportState { @@ -347,6 +350,18 @@ export class SVGViewport extends React.Component - {children(this.state)} + {children(this.state, viewport)} {interactor.render && interactor.render(this)} diff --git a/js_modules/dagit/packages/core/src/pipelines/GraphExplorer.tsx b/js_modules/dagit/packages/core/src/pipelines/GraphExplorer.tsx index 8004ce3a863f8..4e06cd9d42926 100644 --- a/js_modules/dagit/packages/core/src/pipelines/GraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/pipelines/GraphExplorer.tsx @@ -60,7 +60,7 @@ export const GraphExplorer: React.FC = (props) => { repoAddress, isGraph, } = props; - const [highlighted, setHighlighted] = React.useState(''); + const [nameMatch, setNameMatch] = React.useState(''); const handleQueryChange = (opsQuery: string) => { onChangeExplorerPath({...explorerPath, opsQuery}, 'replace'); @@ -138,8 +138,8 @@ export const GraphExplorer: React.FC = (props) => { ); const {all} = queryResultOps; - const highlightedOps = React.useMemo(() => all.filter((s) => s.name.includes(highlighted)), [ - highlighted, + const highlightedOps = React.useMemo(() => all.filter((s) => s.name.includes(nameMatch)), [ + nameMatch, all, ]); @@ -188,9 +188,9 @@ export const GraphExplorer: React.FC = (props) => { setHighlighted(e.target.value)} + onChange={(e) => setNameMatch(e.target.value)} /> {explodeCompositesEnabled && ( @@ -294,7 +294,7 @@ const OptionsOverlay = styled.div` left: 0; `; -const SearchOverlay = styled.div` +export const SearchOverlay = styled.div` z-index: 2; padding: 12px 12px 0 0; display: inline-flex; diff --git a/js_modules/dagit/packages/core/src/pipelines/PipelineExplorerRoot.tsx b/js_modules/dagit/packages/core/src/pipelines/PipelineExplorerRoot.tsx index 1b39d15a369f6..3cbe463abd25f 100644 --- a/js_modules/dagit/packages/core/src/pipelines/PipelineExplorerRoot.tsx +++ b/js_modules/dagit/packages/core/src/pipelines/PipelineExplorerRoot.tsx @@ -134,9 +134,9 @@ export const PipelineExplorerContainer: React.FC<{ ); } diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index a3772cc3e08ff..b56012cb45e36 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -1,28 +1,31 @@ import {gql, useQuery} from '@apollo/client'; +import {uniq, without} from 'lodash'; import React from 'react'; import styled from 'styled-components/macro'; +import {filterByQuery} from '../../app/GraphQueryImpl'; import {LATEST_MATERIALIZATION_METADATA_FRAGMENT} from '../../assets/LastMaterializationMetadata'; +import {LaunchRootExecutionButton} from '../../execute/LaunchRootExecutionButton'; import {SVGViewport} from '../../graph/SVGViewport'; import {useDocumentTitle} from '../../hooks/useDocumentTitle'; import {ExplorerPath} from '../../pipelines/PipelinePathUtils'; import {SidebarPipelineOrJobOverview} from '../../pipelines/SidebarPipelineOrJobOverview'; import {GraphExplorerSolidHandleFragment} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; -import {METADATA_ENTRY_FRAGMENT} from '../../runs/MetadataEntry'; -import {ColorsWIP} from '../../ui/Colors'; +import {GraphQueryInput} from '../../ui/GraphQueryInput'; import {Loading} from '../../ui/Loading'; import {NonIdealState} from '../../ui/NonIdealState'; import {SplitPanelContainer} from '../../ui/SplitPanelContainer'; import {repoAddressToSelector} from '../repoAddressToSelector'; import {RepoAddress} from '../types'; -import {AssetNode, getNodeDimensions} from './AssetNode'; +import {AssetLinks} from './AssetLinks'; +import {AssetNode, ASSET_NODE_FRAGMENT, getNodeDimensions} from './AssetNode'; import {ForeignNode, getForeignNodeDimensions} from './ForeignNode'; import {SidebarAssetInfo} from './SidebarAssetInfo'; import { buildGraphComputeStatuses, buildGraphData, - buildSVGPath, + GraphData, graphHasCycles, layoutGraph, Node, @@ -44,27 +47,12 @@ interface Props { } export const AssetGraphExplorer: React.FC = (props) => { - const {repoAddress, handles, selectedHandle, explorerPath, onChangeExplorerPath} = props; - const repositorySelector = repoAddressToSelector(repoAddress); + const repositorySelector = repoAddressToSelector(props.repoAddress); const queryResult = useQuery(ASSETS_GRAPH_QUERY, { variables: {repositorySelector}, notifyOnNetworkStatusChange: true, }); - const selectNode = React.useCallback( - (node: Node | null) => { - onChangeExplorerPath( - { - ...explorerPath, - opNames: node ? [node.definition.opName!] : [], - pipelineName: node?.definition.jobName || explorerPath.pipelineName, - }, - 'replace', - ); - }, - [onChangeExplorerPath, explorerPath], - ); - useDocumentTitle('Assets'); return ( @@ -74,10 +62,9 @@ export const AssetGraphExplorer: React.FC = (props) => { return ; } - const graphData = buildGraphData(repositoryOrError, explorerPath.pipelineName); - const hasCycles = graphHasCycles(graphData); + const graphData = buildGraphData(repositoryOrError, props.explorerPath.pipelineName); - if (hasCycles) { + if (graphHasCycles(graphData)) { return ( = (props) => { /> ); } - - const layout = layoutGraph(graphData); - const computeStatuses = buildGraphComputeStatuses(graphData); - const selectedAsset = repositoryOrError.assetNodes.find( - (a) => a.opName === selectedHandle?.handleID, - ); - if (!Object.keys(graphData.nodes).length) { return ( = (props) => { ); } - return ( - {}} - onClick={() => selectNode(null)} - maxZoom={1.2} - maxAutocenterZoom={1.0} - > - {({scale: _scale}: any) => ( - - - - - - - - {layout.edges.map((edge, idx) => ( - - ))} - - {layout.nodes.map((layoutNode) => { - const graphNode = graphData.nodes[layoutNode.id]; - const {width, height} = graphNode.hidden - ? getForeignNodeDimensions(layoutNode.id) - : getNodeDimensions(graphNode.definition); - return ( - { - selectNode(graphNode); - e.stopPropagation(); - }} - > - {graphNode.hidden ? ( - - ) : ( - h.handleID === graphNode.definition.opName)! - } - secondaryHighlight={false} - selected={selectedAsset === graphNode.definition} - computeStatus={computeStatuses[graphNode.id]} - repoAddress={repoAddress} - /> - )} - - ); - })} - - )} - - } - second={ - selectedAsset && selectedHandle ? ( - - ) : ( - + return ; + }} + + ); +}; + +const AssetGraphExplorerWithData: React.FC< + {graphData: ReturnType} & Props +> = (props) => { + const { + repoAddress, + handles, + selectedHandle, + explorerPath, + onChangeExplorerPath, + graphData, + } = props; + + const selectedDefinition = selectedHandle?.solid.definition; + const selectedGraphNode = + selectedDefinition && + Object.values(graphData.nodes).find( + (node) => node.definition.opName === selectedDefinition.name, + ); + + const onSelectNode = React.useCallback( + (e: React.MouseEvent, node: Node) => { + e.stopPropagation(); + + const {opName, jobName} = node.definition; + if (!opName) { + return; + } + + const append = jobName === explorerPath.pipelineName && (e.shiftKey || e.metaKey); + const existing = explorerPath.opsQuery.split(','); + const added = + e.shiftKey && selectedGraphNode + ? opsInRange({graph: graphData, from: selectedGraphNode, to: node}) + : [opName]; + + const next = append + ? (existing.includes(opName) + ? without(existing, opName) + : uniq([...existing, ...added]) + ).join(',') + : `${opName}`; + + onChangeExplorerPath( + { + ...explorerPath, + opNames: [opName], + opsQuery: next, + pipelineName: jobName || explorerPath.pipelineName, + }, + 'replace', + ); + }, + [onChangeExplorerPath, explorerPath], + ); + + const {all: highlighted} = React.useMemo( + () => + filterByQuery( + handles.map((h) => h.solid), + explorerPath.opsQuery, + ), + [explorerPath.opsQuery, handles], + ); + + const layout = layoutGraph(graphData); + const computeStatuses = buildGraphComputeStatuses(graphData); + + return ( + + {}} + onClick={() => + onChangeExplorerPath( + { + ...explorerPath, + pipelineName: explorerPath.pipelineName, + opsQuery: '', + opNames: [], + }, + 'replace', ) } + maxZoom={1.2} + maxAutocenterZoom={1.0} + > + {({scale: _scale}, bounds) => ( + + + + {layout.nodes.map((layoutNode) => { + const graphNode = graphData.nodes[layoutNode.id]; + const {width, height} = graphNode.hidden + ? getForeignNodeDimensions(layoutNode.id) + : getNodeDimensions(graphNode.definition); + + if ( + layoutNode.x + width < bounds.left || + layoutNode.y + height < bounds.top || + layoutNode.x > bounds.right || + layoutNode.y > bounds.bottom + ) { + return null; + } + + return ( + onSelectNode(e, graphNode)} + style={{overflow: 'visible'}} + > + {graphNode.hidden ? ( + + ) : ( + h.handleID === graphNode.definition.opName)!.solid + .definition.metadata + } + selected={selectedGraphNode === graphNode} + computeStatus={computeStatuses[graphNode.id]} + repoAddress={repoAddress} + secondaryHighlight={ + explorerPath.opsQuery + ? highlighted.some( + (h) => h.definition.name === graphNode.definition.opName, + ) + : false + } + /> + )} + + ); + })} + + )} + + + + h.solid)} + value={explorerPath.opsQuery} + placeholder="Type an asset subset…" + onChange={(opsQuery) => onChangeExplorerPath({...explorerPath, opsQuery}, 'replace')} + /> + ({ + executionParams: { + mode: 'default', + executionMetadata: {}, + runConfigData: {}, + selector: { + ...repoAddressToSelector(repoAddress), + pipelineName: explorerPath.pipelineName, + solidSelection: highlighted.map((h) => h.name), + }, + }, + })} + /> + + + } + second={ + selectedGraphNode && selectedDefinition ? ( + - ); - }} - + ) : ( + + ) + } + /> ); }; @@ -214,13 +291,11 @@ const ASSETS_GRAPH_QUERY = gql` name } assetNodes { + ...AssetNodeFragment id assetKey { path } - opName - description - jobName dependencies { inputName upstreamAsset { @@ -230,31 +305,6 @@ const ASSETS_GRAPH_QUERY = gql` } } } - assetMaterializations(limit: 1) { - ...LatestMaterializationMetadataFragment - - materializationEvent { - materialization { - metadataEntries { - ...MetadataEntryFragment - } - } - stepStats { - stepKey - startTime - endTime - } - } - runOrError { - ... on PipelineRun { - id - runId - status - pipelineName - mode - } - } - } } pipelines { id @@ -267,7 +317,7 @@ const ASSETS_GRAPH_QUERY = gql` } } } - ${METADATA_ENTRY_FRAGMENT} + ${ASSET_NODE_FRAGMENT} ${LATEST_MATERIALIZATION_METADATA_FRAGMENT} `; @@ -275,9 +325,42 @@ const SVGContainer = styled.svg` overflow: visible; border-radius: 0; `; -const StyledPath = styled('path')<{dashed: boolean}>` - stroke-width: 4; - stroke: ${ColorsWIP.Gray600}; - ${({dashed}) => (dashed ? `stroke-dasharray: 8 2;` : '')} - fill: none; + +const AssetQueryInputContainer = styled.div` + z-index: 2; + position: absolute; + bottom: 10px; + left: 50%; + transform: translateX(-50%); + white-space: nowrap; + display: flex; `; + +const opsInRange = ( + {graph, from, to}: {graph: GraphData; from: Node; to: Node}, + seen: string[] = [], +) => { + if (!from) { + return []; + } + if (from.id === to.id) { + return [to.definition.opName!]; + } + const adjacent = [ + ...Object.keys(graph.upstream[from.id] || {}), + ...Object.keys(graph.downstream[from.id] || {}), + ].map((n) => graph.nodes[n]); + + let best: string[] = []; + + for (const node of adjacent) { + if (seen.includes(node.id)) { + continue; + } + const result: string[] = opsInRange({graph, from: node, to}, [...seen, from.id]); + if (result.length && (best.length === 0 || result.length < best.length)) { + best = [from.definition.opName!, ...result]; + } + } + return best; +}; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetLinks.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetLinks.tsx new file mode 100644 index 0000000000000..bfda4699ead88 --- /dev/null +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetLinks.tsx @@ -0,0 +1,39 @@ +import React from 'react'; +import styled from 'styled-components/macro'; + +import {ColorsWIP} from '../../ui/Colors'; + +import {buildSVGPath, IEdge} from './Utils'; + +export const AssetLinks: React.FC<{edges: IEdge[]}> = React.memo(({edges}) => ( + <> + + + + + + {edges.map((edge, idx) => ( + + ))} + +)); + +const StyledPath = styled('path')<{dashed: boolean}>` + stroke-width: 4; + stroke: ${ColorsWIP.KeylineGray}; + ${({dashed}) => (dashed ? `stroke-dasharray: 8 2;` : '')} + fill: none; +`; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx index 9607054a0e03b..7f7a6d78b7cbe 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx @@ -1,14 +1,16 @@ -import {useMutation} from '@apollo/client'; +import {gql, useMutation} from '@apollo/client'; import {ContextMenu2 as ContextMenu} from '@blueprintjs/popover2'; +import {isEqual} from 'lodash'; import qs from 'query-string'; import React, {CSSProperties} from 'react'; import {Link} from 'react-router-dom'; import styled from 'styled-components/macro'; import {AppContext} from '../../app/AppContext'; +import {LATEST_MATERIALIZATION_METADATA_FRAGMENT} from '../../assets/LastMaterializationMetadata'; import {showLaunchError} from '../../execute/showLaunchError'; import {OpTags} from '../../graph/OpTags'; -import {GraphExplorerSolidHandleFragment} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; +import {METADATA_ENTRY_FRAGMENT} from '../../runs/MetadataEntry'; import { LAUNCH_PIPELINE_EXECUTION_MUTATION, handleLaunchResult, @@ -27,168 +29,214 @@ import {RepoAddress} from '../types'; import {workspacePath} from '../workspacePath'; import {assetKeyToString, Status} from './Utils'; -import {AssetGraphQuery_repositoryOrError_Repository_assetNodes} from './types/AssetGraphQuery'; +import {AssetNodeFragment} from './types/AssetNodeFragment'; export const AssetNode: React.FC<{ - definition: AssetGraphQuery_repositoryOrError_Repository_assetNodes; - handle: GraphExplorerSolidHandleFragment; + definition: AssetNodeFragment; + metadata: {key: string; value: string}[]; selected: boolean; computeStatus: Status; repoAddress: RepoAddress; secondaryHighlight: boolean; -}> = ({definition, handle, selected, computeStatus, repoAddress, secondaryHighlight}) => { - const [launchPipelineExecution] = useMutation( - LAUNCH_PIPELINE_EXECUTION_MUTATION, - ); - const {basePath} = React.useContext(AppContext); - const {materializationEvent: event, runOrError} = definition.assetMaterializations[0] || {}; - const kind = handle.solid.definition.metadata.find((m) => m.key === 'kind')?.value; +}> = React.memo( + ({definition, metadata, selected, computeStatus, repoAddress, secondaryHighlight}) => { + const [launchPipelineExecution] = useMutation( + LAUNCH_PIPELINE_EXECUTION_MUTATION, + ); + const {basePath} = React.useContext(AppContext); + const {materializationEvent: event, runOrError} = definition.assetMaterializations[0] || {}; + const kind = metadata.find((m) => m.key === 'kind')?.value; - const onLaunch = async () => { - if (!definition.jobName) { - return; - } + const onLaunch = async () => { + if (!definition.jobName) { + return; + } - try { - const result = await launchPipelineExecution({ - variables: { - executionParams: { - selector: { - pipelineName: definition.jobName, - ...repoAddressToSelector(repoAddress), + try { + const result = await launchPipelineExecution({ + variables: { + executionParams: { + selector: { + pipelineName: definition.jobName, + ...repoAddressToSelector(repoAddress), + }, + mode: 'default', + stepKeys: [definition.opName], }, - mode: 'default', - stepKeys: [definition.opName], }, - }, - }); - handleLaunchResult(basePath, definition.jobName, result, true); - } catch (error) { - showLaunchError(error as Error); - } - }; + }); + handleLaunchResult(basePath, definition.jobName, result, true); + } catch (error) { + showLaunchError(error as Error); + } + }; - return ( - - - Launch run to build{' '} - - {assetKeyToString(definition.assetKey)} + return ( + + + Launch run to build{' '} + + {assetKeyToString(definition.assetKey)} + - - } - icon="open_in_new" - onClick={onLaunch} - /> - - } - > - - - - - {assetKeyToString(definition.assetKey)} -
- {computeStatus === 'old' && ( - - upstream -
- changed - - )} - - {definition.description && ( - {markdownToPlaintext(definition.description).split('\n')[0]} - )} - {event ? ( - - {runOrError.__typename === 'Run' && ( - - - {`${runOrError.pipelineName}${ - runOrError.mode !== 'default' ? `:${runOrError.mode}` : '' - }`} - - - {titleForRun({runId: runOrError.runId})} - - + } + icon="open_in_new" + onClick={onLaunch} + /> + + } + > + + + + + {assetKeyToString(definition.assetKey)} +
+ {computeStatus === 'old' && ( + + upstream +
+ changed + )} + + {definition.description && ( + + {markdownToPlaintext(definition.description).split('\n')[0]} + + )} + {event ? ( + + {runOrError.__typename === 'Run' && ( + + + {`${runOrError.pipelineName}${ + runOrError.mode !== 'default' ? `:${runOrError.mode}` : '' + }`} + + + {titleForRun({runId: runOrError.runId})} + + + )} - - {event.stepStats.endTime ? ( - + {event.stepStats.endTime ? ( + + ) : ( + 'Never' + )} + - ) : ( - 'Never' - )} - - - - ) : ( - - - No materializations - — - - - — - — - - - )} - {kind && ( - { - window.requestAnimationFrame(() => - document.dispatchEvent(new Event('show-kind-info')), - ); + + + ) : ( + + + No materializations + — + + + — + — + + + )} + {kind && ( + { + window.requestAnimationFrame(() => + document.dispatchEvent(new Event('show-kind-info')), + ); + }, }, - }, - ]} - /> - )} - - - - ); -}; + ]} + /> + )} + + + + ); + }, + isEqual, +); + +export const ASSET_NODE_FRAGMENT = gql` + fragment AssetNodeFragment on AssetNode { + id + opName + description + jobName + assetKey { + path + } + + assetMaterializations(limit: 1) { + ...LatestMaterializationMetadataFragment + + materializationEvent { + materialization { + metadataEntries { + ...MetadataEntryFragment + } + } + stepStats { + stepKey + startTime + endTime + } + } + runOrError { + ... on PipelineRun { + id + runId + status + pipelineName + mode + } + } + } + } + + ${LATEST_MATERIALIZATION_METADATA_FRAGMENT} + ${METADATA_ENTRY_FRAGMENT} +`; -export const getNodeDimensions = (def: AssetGraphQuery_repositoryOrError_Repository_assetNodes) => { - let height = 92 + 20; +export const getNodeDimensions = (def: AssetNodeFragment) => { + let height = 95; if (def.description) { height += 25; } @@ -211,7 +259,7 @@ const AssetNodeContainer = styled.div<{$selected: boolean; $secondaryHighlight: p.$selected ? `2px dashed rgba(255, 69, 0, 1)` : p.$secondaryHighlight - ? `2px solid ${ColorsWIP.Blue500}55` + ? `2px dashed rgba(255, 69, 0, 0.5)` : 'none'}; border-radius: 6px; outline-offset: -1px; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx index d00e74cbe68e8..8182b5bb712da 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx @@ -1,3 +1,4 @@ +import {isEqual} from 'lodash'; import React from 'react'; import styled from 'styled-components/macro'; @@ -7,12 +8,12 @@ import {FontFamily} from '../../ui/styles'; import {assetKeyToString} from './Utils'; -export const ForeignNode: React.FC<{assetKey: {path: string[]}}> = ({assetKey}) => ( +export const ForeignNode: React.FC<{assetKey: {path: string[]}}> = React.memo(({assetKey}) => ( {assetKeyToString(assetKey)} -); +)); const ForeignNodeLink = styled.div` display: flex; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index 4b5522ffc5d96..8476a7c9a9593 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -6,7 +6,7 @@ import {AssetMaterializations} from '../../assets/AssetMaterializations'; import {LatestMaterializationMetadata} from '../../assets/LastMaterializationMetadata'; import {Description} from '../../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../../pipelines/SidebarComponents'; -import {GraphExplorerSolidHandleFragment} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; +import {GraphExplorerSolidHandleFragment_solid_definition} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; import {pluginForMetadata} from '../../plugins'; import {Box} from '../../ui/Box'; import {ColorsWIP} from '../../ui/Colors'; @@ -18,10 +18,9 @@ import {AssetGraphQuery_repositoryOrError_Repository_assetNodes} from './types/A export const SidebarAssetInfo: React.FC<{ node: AssetGraphQuery_repositoryOrError_Repository_assetNodes; - handle: GraphExplorerSolidHandleFragment; + definition: GraphExplorerSolidHandleFragment_solid_definition; repoAddress: RepoAddress; -}> = ({node, handle, repoAddress}) => { - const definition = handle.solid.definition; +}> = ({node, definition, repoAddress}) => { const Plugin = pluginForMetadata(definition.metadata); return ( @@ -39,14 +38,17 @@ export const SidebarAssetInfo: React.FC<{
{node.assetMaterializations.length ? ( - - - - - {'View All in Asset Catalog '} - - - + <> +
+ +
+ + + {'View All in Asset Catalog '} + + + + ) : ( — )} @@ -54,9 +56,7 @@ export const SidebarAssetInfo: React.FC<{ {node.assetMaterializations.length ? ( - - - + ) : null}
diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx index 5fa8bc3ab57d4..aed768ebd30bb 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx @@ -1,5 +1,6 @@ import {pathVerticalDiagonal} from '@vx/shape'; import * as dagre from 'dagre'; +import memoize from 'lodash/memoize'; import {AssetNode, getNodeDimensions} from './AssetNode'; import {getForeignNodeDimensions} from './ForeignNode'; @@ -24,7 +25,7 @@ interface LayoutNode { x: number; y: number; } -interface GraphData { +export interface GraphData { nodes: {[id: string]: Node}; downstream: {[upstream: string]: {[downstream: string]: string}}; upstream: {[downstream: string]: {[upstream: string]: boolean}}; @@ -50,11 +51,11 @@ export const buildGraphData = (repository: Repository, jobName?: string) => { repository.assetNodes.forEach((definition: AssetNode) => { const assetKeyJson = JSON.stringify(definition.assetKey.path); - definition.dependencies.forEach((dependency) => { - const upstreamAssetKeyJson = JSON.stringify(dependency.upstreamAsset.assetKey.path); + definition.dependencies.forEach(({asset, inputName}) => { + const upstreamAssetKeyJson = JSON.stringify(asset.assetKey.path); downstream[upstreamAssetKeyJson] = { ...(downstream[upstreamAssetKeyJson] || {}), - [assetKeyJson]: dependency.inputName, + [assetKeyJson]: inputName, }; upstream[assetKeyJson] = { ...(upstream[assetKeyJson] || {}), @@ -93,12 +94,10 @@ export const graphHasCycles = (graphData: GraphData) => { return hasCycles; }; -export const layoutGraph = (graphData: GraphData) => { +export const _layoutGraph = (graphData: GraphData, margin = 100) => { const g = new dagre.graphlib.Graph(); - const marginBase = 100; - const marginy = marginBase; - const marginx = marginBase; - g.setGraph({rankdir: 'TB', marginx, marginy}); + + g.setGraph({rankdir: 'TB', marginx: margin, marginy: margin}); g.setDefaultEdgeLabel(() => ({})); Object.values(graphData.nodes) @@ -147,8 +146,8 @@ export const layoutGraph = (graphData: GraphData) => { x: dagreNode.x - dagreNode.width / 2, y: dagreNode.y - dagreNode.height / 2, }); - maxWidth = Math.max(maxWidth, dagreNode.x + dagreNode.width); - maxHeight = Math.max(maxHeight, dagreNode.y + dagreNode.height); + maxWidth = Math.max(maxWidth, dagreNode.x + dagreNode.width / 2); + maxHeight = Math.max(maxHeight, dagreNode.y + dagreNode.height / 2); }); const edges: IEdge[] = []; @@ -164,8 +163,8 @@ export const layoutGraph = (graphData: GraphData) => { return { nodes, edges, - width: maxWidth, - height: maxHeight + marginBase, + width: maxWidth + margin, + height: maxHeight + margin, }; }; @@ -206,6 +205,12 @@ export function buildGraphComputeStatuses(graphData: GraphData) { return statuses; } +const _layoutCacheKey = (data: GraphData) => { + return Object.keys(data.nodes).join('|'); +}; + +export const layoutGraph = memoize(_layoutGraph, _layoutCacheKey); + export type Status = 'good' | 'old' | 'none'; function findComputeStatusForId( diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts index f4de51e4c6575..4fa84adc101ad 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts @@ -24,23 +24,6 @@ export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetKe path: string[]; } -export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset_assetKey { - __typename: "AssetKey"; - path: string[]; -} - -export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset { - __typename: "AssetNode"; - id: string; - assetKey: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset_assetKey; -} - -export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies { - __typename: "AssetDependency"; - inputName: string; - upstreamAsset: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset; -} - export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetMaterializations_runOrError_RunNotFoundError { __typename: "RunNotFoundError" | "PythonError"; } @@ -184,15 +167,32 @@ export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetMa materializationEvent: AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetMaterializations_materializationEvent; } +export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset { + __typename: "AssetNode"; + id: string; + assetKey: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset_assetKey; +} + +export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies { + __typename: "AssetDependency"; + inputName: string; + upstreamAsset: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset; +} + export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes { __typename: "AssetNode"; id: string; - assetKey: AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetKey; opName: string | null; description: string | null; jobName: string | null; - dependencies: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies[]; + assetKey: AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetKey; assetMaterializations: AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetMaterializations[]; + dependencies: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies[]; } export interface AssetGraphQuery_repositoryOrError_Repository_pipelines_modes { diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts new file mode 100644 index 0000000000000..8f19ca5ab5a91 --- /dev/null +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts @@ -0,0 +1,168 @@ +/* tslint:disable */ +/* eslint-disable */ +// @generated +// This file was automatically generated and should not be edited. + +import { RunStatus } from "./../../../types/globalTypes"; + +// ==================================================== +// GraphQL fragment: AssetNodeFragment +// ==================================================== + +export interface AssetNodeFragment_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeFragment_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetNodeFragment_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetNodeFragment_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetNodeFragment_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetNodeFragment_assetMaterializations_runOrError = AssetNodeFragment_assetMaterializations_runOrError_RunNotFoundError | AssetNodeFragment_assetMaterializations_runOrError_Run; + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetNodeFragment_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetNodeFragment_assetMaterializations_materializationEvent_stepStats; + materialization: AssetNodeFragment_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetNodeFragment_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetNodeFragment_assetMaterializations_runOrError; + materializationEvent: AssetNodeFragment_assetMaterializations_materializationEvent; +} + +export interface AssetNodeFragment { + __typename: "AssetNode"; + id: string; + opName: string | null; + description: string | null; + jobName: string | null; + assetKey: AssetNodeFragment_assetKey; + assetMaterializations: AssetNodeFragment_assetMaterializations[]; +} From aa509880037df1f84b80c145f64c825df55dc961 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 22 Nov 2021 16:19:41 -0600 Subject: [PATCH 02/22] Fix memoization --- .../asset-graph/AssetGraphExplorer.tsx | 22 ++++++++++++++----- .../core/src/workspace/asset-graph/Utils.tsx | 15 ++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index b56012cb45e36..7bf65bce585e8 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -47,7 +47,8 @@ interface Props { } export const AssetGraphExplorer: React.FC = (props) => { - const repositorySelector = repoAddressToSelector(props.repoAddress); + const {repoAddress, explorerPath} = props; + const repositorySelector = repoAddressToSelector(repoAddress); const queryResult = useQuery(ASSETS_GRAPH_QUERY, { variables: {repositorySelector}, notifyOnNetworkStatusChange: true, @@ -55,15 +56,24 @@ export const AssetGraphExplorer: React.FC = (props) => { useDocumentTitle('Assets'); + const graphData = React.useMemo(() => { + const repositoryOrError = + queryResult.data?.repositoryOrError.__typename === 'Repository' + ? queryResult.data?.repositoryOrError + : null; + if (!repositoryOrError) { + return null; + } + return buildGraphData(repositoryOrError, explorerPath.pipelineName); + }, [queryResult, explorerPath.pipelineName]); + return ( {({repositoryOrError}) => { - if (repositoryOrError.__typename !== 'Repository') { + if (repositoryOrError.__typename !== 'Repository' || !graphData) { return ; } - const graphData = buildGraphData(repositoryOrError, props.explorerPath.pipelineName); - if (graphHasCycles(graphData)) { return ( layoutGraph(graphData), [graphData]); + const computeStatuses = React.useMemo(() => buildGraphComputeStatuses(graphData), [graphData]); return ( { repository.assetNodes.forEach((definition: AssetNode) => { const assetKeyJson = JSON.stringify(definition.assetKey.path); - definition.dependencies.forEach(({asset, inputName}) => { - const upstreamAssetKeyJson = JSON.stringify(asset.assetKey.path); + definition.dependencies.forEach(({upstreamAsset, inputName}) => { + const upstreamAssetKeyJson = JSON.stringify(upstreamAsset.assetKey.path); downstream[upstreamAssetKeyJson] = { ...(downstream[upstreamAssetKeyJson] || {}), [assetKeyJson]: inputName, @@ -94,7 +93,7 @@ export const graphHasCycles = (graphData: GraphData) => { return hasCycles; }; -export const _layoutGraph = (graphData: GraphData, margin = 100) => { +export const layoutGraph = (graphData: GraphData, margin = 100) => { const g = new dagre.graphlib.Graph(); g.setGraph({rankdir: 'TB', marginx: margin, marginy: margin}); @@ -205,12 +204,6 @@ export function buildGraphComputeStatuses(graphData: GraphData) { return statuses; } -const _layoutCacheKey = (data: GraphData) => { - return Object.keys(data.nodes).join('|'); -}; - -export const layoutGraph = memoize(_layoutGraph, _layoutCacheKey); - export type Status = 'good' | 'old' | 'none'; function findComputeStatusForId( From 89c6c2778c644468bbfdb60b298c195d908eaaa5 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 22 Nov 2021 16:24:11 -0600 Subject: [PATCH 03/22] Fix linter issues --- js_modules/dagit/packages/core/src/graph/OpLinks.tsx | 1 - .../core/src/workspace/asset-graph/AssetGraphExplorer.tsx | 2 +- .../packages/core/src/workspace/asset-graph/ForeignNode.tsx | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/js_modules/dagit/packages/core/src/graph/OpLinks.tsx b/js_modules/dagit/packages/core/src/graph/OpLinks.tsx index 89cc6b657a280..55e575a73749b 100644 --- a/js_modules/dagit/packages/core/src/graph/OpLinks.tsx +++ b/js_modules/dagit/packages/core/src/graph/OpLinks.tsx @@ -3,7 +3,6 @@ import * as React from 'react'; import styled from 'styled-components/macro'; import {weakmapMemoize} from '../app/Util'; -import {ColorsWIP} from '../ui/Colors'; import {IFullPipelineLayout, IFullOpLayout, ILayoutConnection} from './getFullOpLayout'; import {PipelineGraphOpFragment} from './types/PipelineGraphOpFragment'; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index 7bf65bce585e8..dfcb00af0c695 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -151,7 +151,7 @@ const AssetGraphExplorerWithData: React.FC< 'replace', ); }, - [onChangeExplorerPath, explorerPath], + [explorerPath, selectedGraphNode, graphData, onChangeExplorerPath], ); const {all: highlighted} = React.useMemo( diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx index 8182b5bb712da..ac9d511eea7ea 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx @@ -1,4 +1,3 @@ -import {isEqual} from 'lodash'; import React from 'react'; import styled from 'styled-components/macro'; From 806bb11ad29ef16fa40ac364c053dbf7cb365cce Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 22 Nov 2021 16:38:26 -0600 Subject: [PATCH 04/22] [dagit] Reworked asset details page # Conflicts: # js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx # js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx # js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts --- .../core/src/assets/AssetEntryRoot.tsx | 26 +- .../src/assets/AssetMaterializationMatrix.tsx | 332 ------------ .../src/assets/AssetMaterializationTable.tsx | 188 ++++--- .../core/src/assets/AssetMaterializations.tsx | 174 +++---- .../core/src/assets/AssetNeighborsGraph.tsx | 126 +++++ .../core/src/assets/AssetNodeDefinition.tsx | 65 ++- .../packages/core/src/assets/AssetView.tsx | 46 +- .../core/src/assets/SnapshotWarning.tsx | 78 --- .../packages/core/src/assets/Sparkline.tsx | 65 --- .../types/AssetNodeDefinitionFragment.ts | 482 ++++++++++++++++++ .../core/src/assets/types/AssetQuery.ts | 403 +++++++++++++-- .../types/SnapshotWarningAssetFragment.ts | 41 -- .../src/assets/useMaterializationBuckets.tsx | 8 +- .../packages/core/src/graphql/schema.graphql | 3 +- .../asset-graph/AssetGraphExplorer.tsx | 4 +- .../src/workspace/asset-graph/ForeignNode.tsx | 1 + .../core/src/workspace/asset-graph/Utils.tsx | 4 +- .../asset-graph/types/AssetGraphQuery.ts | 8 +- .../dagster_graphql/schema/asset_graph.py | 28 +- 19 files changed, 1259 insertions(+), 823 deletions(-) delete mode 100644 js_modules/dagit/packages/core/src/assets/AssetMaterializationMatrix.tsx create mode 100644 js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx delete mode 100644 js_modules/dagit/packages/core/src/assets/SnapshotWarning.tsx delete mode 100644 js_modules/dagit/packages/core/src/assets/Sparkline.tsx delete mode 100644 js_modules/dagit/packages/core/src/assets/types/SnapshotWarningAssetFragment.ts diff --git a/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx b/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx index b23ac16279fa6..9b4052f590937 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx @@ -1,7 +1,7 @@ import {gql, useQuery} from '@apollo/client'; import {BreadcrumbProps, Breadcrumbs} from '@blueprintjs/core'; import * as React from 'react'; -import {Link, Redirect, RouteComponentProps} from 'react-router-dom'; +import {Link, RouteComponentProps} from 'react-router-dom'; import styled from 'styled-components/macro'; import {Box} from '../ui/Box'; @@ -17,27 +17,12 @@ import {AssetsCatalogTable} from './AssetsCatalogTable'; import {AssetEntryRootQuery} from './types/AssetEntryRootQuery'; import {useAssetView} from './useAssetView'; -// Jan 1, 2015 at 00:00 GMT -const EARLIEST_TIME = 1420070400000; - -export const AssetEntryRoot: React.FC = ({location, match}) => { +export const AssetEntryRoot: React.FC = ({match}) => { const currentPath: string[] = (match.params['0'] || '') .split('/') .filter((x: string) => x) .map(decodeURIComponent); - const {pathname, search} = location; - const asOf = React.useMemo(() => { - const params = new URLSearchParams(search); - return params.get('asOf'); - }, [search]); - - // Validate the `asOf` time, since it's user-specified. - const invalidTime = React.useMemo(() => { - const asOfNumber = Number(asOf); - return asOfNumber && (asOfNumber < EARLIEST_TIME || asOfNumber > Date.now()); - }, [asOf]); - const [view] = useAssetView(); const queryResult = useQuery(ASSET_ENTRY_ROOT_QUERY, { @@ -59,11 +44,6 @@ export const AssetEntryRoot: React.FC = ({location, match}) return list; }, [currentPath, view]); - // If the asOf timestamp is invalid, discard it via redirect. - if (invalidTime) { - return ; - } - return ( = ({location, match}) return ; } - return ; + return ; }} diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializationMatrix.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializationMatrix.tsx deleted file mode 100644 index f8cb71d540bfc..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializationMatrix.tsx +++ /dev/null @@ -1,332 +0,0 @@ -import flatMap from 'lodash/flatMap'; -import uniq from 'lodash/uniq'; -import * as React from 'react'; -import {Link} from 'react-router-dom'; -import styled from 'styled-components/macro'; - -import {formatElapsedTime} from '../app/Util'; -import {Timestamp} from '../app/time/Timestamp'; -import {useViewport} from '../gantt/useViewport'; -import { - GridColumn, - GridFloatingContainer, - GridScrollContainer, - LeftLabel, -} from '../partitions/RunMatrixUtils'; -import {MetadataEntry} from '../runs/MetadataEntry'; -import {titleForRun} from '../runs/RunUtils'; -import {MetadataEntryFragment} from '../runs/types/MetadataEntryFragment'; -import {Box} from '../ui/Box'; -import {ColorsWIP} from '../ui/Colors'; -import {IconWIP, IconWrapper} from '../ui/Icon'; -import {FontFamily} from '../ui/styles'; - -import {AssetPredecessorLink} from './AssetMaterializationTable'; -import {Sparkline} from './Sparkline'; -import {AssetNumericHistoricalData} from './types'; -import {AssetMaterializationFragment} from './types/AssetMaterializationFragment'; -import {HistoricalMaterialization} from './useMaterializationBuckets'; - -const COL_WIDTH = 240; - -const OVERSCROLL = 150; - -export const LABEL_STEP_EXECUTION_TIME = 'Step Execution Time'; - -interface AssetMaterializationMatrixProps { - materializations: HistoricalMaterialization[]; - isPartitioned: boolean; - xAxis: 'time' | 'partition'; - xHover: number | string | null; - onHoverX: (x: number | string | null) => void; - graphDataByMetadataLabel: AssetNumericHistoricalData; - graphedLabels: string[]; - setGraphedLabels: (labels: string[]) => void; -} - -function xForAssetMaterialization(am: AssetMaterializationFragment, xAxis: 'time' | 'partition') { - return xAxis === 'time' ? Number(am.materializationEvent.timestamp) : am.partition; -} - -const EMPTY_CELL_DASH = - ; - -export const AssetMaterializationMatrix: React.FC = ({ - materializations, - isPartitioned, - xAxis, - xHover, - onHoverX, - graphDataByMetadataLabel, - graphedLabels, - setGraphedLabels, -}) => { - const [hoveredLabel, setHoveredLabel] = React.useState(''); - const {viewport, containerProps, onMoveToViewport} = useViewport({ - initialOffset: React.useCallback((el) => ({left: el.scrollWidth - el.clientWidth, top: 0}), []), - }); - - const anyPredecessors = materializations.some(({predecessors}) => !!predecessors?.length); - - const visibleRangeStart = Math.max(0, Math.floor((viewport.left - OVERSCROLL) / COL_WIDTH)); - const visibleCount = Math.ceil((viewport.width + OVERSCROLL * 2) / COL_WIDTH); - const visible = materializations.slice(visibleRangeStart, visibleRangeStart + visibleCount); - - const lastXHover = React.useRef(null); - React.useEffect(() => { - if (lastXHover.current === xHover) { - return; - } - if (xHover !== null) { - const idx = materializations.findIndex( - (m) => xForAssetMaterialization(m.latest, xAxis) === xHover, - ); - if ((idx !== -1 && idx < visibleRangeStart) || idx > visibleRangeStart + visibleCount) { - onMoveToViewport({left: idx * COL_WIDTH - (viewport.width - COL_WIDTH) / 2, top: 0}, false); - } - } - lastXHover.current = xHover; - }); - - const metadataLabels = uniq( - flatMap(materializations, (m) => - m.latest.materializationEvent.materialization.metadataEntries.map((e) => e.label), - ), - ); - - return ( - -
- - - {isPartitioned && Partition} - Run - Timestamp - {anyPredecessors ? Previous materializations : null} - - {[...metadataLabels, LABEL_STEP_EXECUTION_TIME].map((label, idx) => ( - - setGraphedLabels( - graphedLabels.includes(label) - ? graphedLabels.filter((l) => l !== label) - : [...graphedLabels, label], - ) - } - /> - ))} - - - -
- {visible.map((historicalMaterialization, visibleIdx) => { - const {latest: assetMaterialization, predecessors} = historicalMaterialization; - const {materializationEvent, partition} = assetMaterialization; - const x = xForAssetMaterialization(assetMaterialization, xAxis); - const {startTime, endTime} = materializationEvent.stepStats || {}; - - const runId = - assetMaterialization.runOrError.__typename === 'Run' - ? assetMaterialization.runOrError.runId - : ''; - - return ( - onHoverX(x)} - hovered={xHover === x} - style={{ - width: COL_WIDTH, - position: 'absolute', - zIndex: visible.length - visibleIdx, - left: (visibleIdx + visibleRangeStart) * COL_WIDTH, - }} - > - {isPartitioned &&
{partition}
} -
- - {titleForRun({runId})} - -
-
- -
- {anyPredecessors ? ( -
- {predecessors?.length ? ( - - ) : null} -
- ) : null} - - {metadataLabels.map((label) => { - const entry = materializationEvent.materialization.metadataEntries.find( - (m) => m.label === label, - ); - return ( -
setHoveredLabel(label)} - onMouseLeave={() => setHoveredLabel('')} - > - {entry ? : EMPTY_CELL_DASH} -
- ); - })} -
setHoveredLabel(LABEL_STEP_EXECUTION_TIME)} - onMouseLeave={() => setHoveredLabel('')} - > - {endTime && startTime - ? formatElapsedTime((endTime - startTime) * 1000) - : EMPTY_CELL_DASH} -
- - ); - })} -
- -
- {visible.length === 0 && No data to display.} - - ); -}; - -const HeaderRowLabel = styled(LeftLabel)` - padding: 4px 24px; - color: ${ColorsWIP.Gray700}; - height: 32px; - box-shadow: ${ColorsWIP.KeylineGray} 0 -1px 0 inset; -`; - -const MetadataRowLabel: React.FunctionComponent<{ - bordered?: boolean; - label: string; - hovered: boolean; - graphEnabled: boolean; - graphData?: AssetNumericHistoricalData[0]; - onToggleGraphEnabled: () => void; -}> = ({label, hovered, graphEnabled, graphData, onToggleGraphEnabled}) => ( - - {graphData ? ( - - - - ) : null} -
- {label} -
- {graphData && } - -); - -const StyledMetadataRowLabel = styled(LeftLabel)` - display: flex; - align-items: center; - border: none; - color: ${ColorsWIP.Gray700}; - height: 32px; - padding: 8px 0 8px 24px; - position: relative; - box-shadow: ${ColorsWIP.KeylineGray} 0 -1px 0 inset; -`; - -const VisibilityButton = styled.button` - background-color: transparent; - border: none; - cursor: pointer; - padding: 0; - margin: 0; - position: absolute; - left: 4px; - - :focus, - :active { - outline: none; - } - - ${IconWrapper} { - transition: background-color 100ms; - } - - :hover ${IconWrapper} { - background-color: ${ColorsWIP.Gray900}; - } -`; - -const AssetGridColumnSpacer = styled.div` - padding: 4px 0; - box-shadow: ${ColorsWIP.KeylineGray} 0 -1px 0 inset, ${ColorsWIP.KeylineGray} -1px 0 0 inset; -`; - -const AssetGridColumn = styled(GridColumn)` - width: 240px; - - .cell { - height: 32px; - padding: 8px; - box-shadow: ${ColorsWIP.KeylineGray} 0 -1px 0 inset, ${ColorsWIP.KeylineGray} -1px 0 0 inset; - font-family: ${FontFamily.monospace}; - font-size: 16px; - } -`; - -const plaintextFor = (entry: MetadataEntryFragment | undefined) => { - if (!entry) { - return ''; - } - if (entry.__typename === 'EventFloatMetadataEntry') { - return entry.floatValue; - } else if (entry.__typename === 'EventIntMetadataEntry') { - if (entry.intValue !== null) { - return entry.intValue.toLocaleString(); - } - return entry.intRepr; - } else if (entry.__typename === 'EventPathMetadataEntry') { - return entry.path; - } else if (entry.__typename === 'EventTextMetadataEntry') { - return entry.text; - } else if (entry.__typename === 'EventUrlMetadataEntry') { - return entry.url; - } else { - return ''; - } -}; - -const EmptyMessage = styled.div` - padding: 20px; - text-align: center; -`; - -const PartitionRunMatrixContainer = styled.div` - display: block; - margin-bottom: 20px; -`; diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx index b3dcea5f46f71..3c9b6ffacee7e 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx @@ -1,10 +1,11 @@ +import qs from 'query-string'; import * as React from 'react'; import {Link} from 'react-router-dom'; import styled from 'styled-components/macro'; import {Timestamp} from '../app/time/Timestamp'; import {PipelineReference} from '../pipelines/PipelineReference'; -import {MetadataEntries} from '../runs/MetadataEntry'; +import {MetadataEntry} from '../runs/MetadataEntry'; import {RunStatusWithStats} from '../runs/RunStatusDots'; import {titleForRun} from '../runs/RunUtils'; import {Box} from '../ui/Box'; @@ -13,6 +14,7 @@ import {ButtonLink} from '../ui/ButtonLink'; import {ColorsWIP} from '../ui/Colors'; import {DialogFooter, DialogWIP} from '../ui/Dialog'; import {Group} from '../ui/Group'; +import {IconWIP} from '../ui/Icon'; import {Table} from '../ui/Table'; import {Mono} from '../ui/Text'; import {isThisThingAJob, useRepository} from '../workspace/WorkspaceContext'; @@ -23,37 +25,31 @@ import {AssetMaterializationFragment} from './types/AssetMaterializationFragment import {HistoricalMaterialization} from './useMaterializationBuckets'; export const AssetMaterializationTable: React.FC<{ - isPartitioned: boolean; + hasPartitions: boolean; hasLineage: boolean; materializations: HistoricalMaterialization[]; - shouldBucketPartitions?: boolean; -}> = ({isPartitioned, hasLineage, materializations, shouldBucketPartitions = true}) => { - const list = React.useMemo(() => { - if (shouldBucketPartitions) { - return materializations; - } - return materializations.map((m) => ({latest: m.latest})); - }, [materializations, shouldBucketPartitions]); - + focused?: string; + setFocused?: (timestamp: string) => void; +}> = ({hasPartitions, hasLineage, materializations, focused, setFocused}) => { return ( - {isPartitioned && } - - {hasLineage && } + {hasPartitions && } - {list.map((m) => ( + {materializations.map((m) => ( ))} @@ -63,9 +59,11 @@ export const AssetMaterializationTable: React.FC<{ const AssetMaterializationRow: React.FC<{ assetMaterialization: HistoricalMaterialization; - isPartitioned: boolean; + hasPartitions: boolean; hasLineage: boolean; -}> = ({assetMaterialization, isPartitioned, hasLineage}) => { + focused?: string; + setFocused?: (timestamp: string) => void; +}> = ({assetMaterialization, hasPartitions, hasLineage, focused, setFocused}) => { const {latest, predecessors} = assetMaterialization; const run = latest.runOrError.__typename === 'Run' ? latest.runOrError : undefined; const repositoryOrigin = run?.repositoryOrigin; @@ -77,85 +75,124 @@ const AssetMaterializationRow: React.FC<{ if (!run) { return ; } - const {materialization, assetLineage, timestamp} = latest.materializationEvent; + const {materialization, assetLineage, timestamp, stepKey} = latest.materializationEvent; const metadataEntries = materialization.metadataEntries; + const isFocused = focused === timestamp; + + const focusCss = isFocused + ? {borderLeft: `5px solid ${ColorsWIP.HighlightGreen}`} + : {paddingLeft: 29}; return ( - - {isPartitioned && ( - setFocused?.(timestamp)}> + {hasPartitions && ( + + )} + + + + + {isFocused && ( + + + + + + ))} + {hasLineage && ( + + + + + )} + + ) : ( + No materialization event metadata + )} + + )} - - {hasLineage && ( - - )} - - - - + ); }; -const MetadataTableContainer = styled.div` - margin-top: -4px; - margin-bottom: -10px; - - & tbody > tr > td { - font-size: 13px; +const DetailsTable = styled.table` + margin: -2px -2px -3px; + tr td { + font-size: 14px; } `; interface PredecessorDialogProps { hasLineage: boolean; - isPartitioned: boolean; + hasPartitions: boolean; predecessors: AssetMaterializationFragment[]; } export const AssetPredecessorLink: React.FC = ({ hasLineage, - isPartitioned, + hasPartitions, predecessors, }) => { const [open, setOpen] = React.useState(false); const count = predecessors.length; const title = () => { - if (isPartitioned) { + if (hasPartitions) { const partition = predecessors[0].partition; if (partition) { return `Previous materializations for ${partition}`; @@ -178,9 +215,8 @@ export const AssetPredecessorLink: React.FC = ({ ({latest: p}))} - shouldBucketPartitions={false} /> diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index 6bd92aef14e4c..0224be5b1cc00 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -10,11 +10,9 @@ import {ButtonGroup} from '../ui/ButtonGroup'; import {ColorsWIP} from '../ui/Colors'; import {NonIdealState} from '../ui/NonIdealState'; import {Spinner} from '../ui/Spinner'; -import {Tab, Tabs} from '../ui/Tabs'; import {Subheading} from '../ui/Text'; import {ASSET_LINEAGE_FRAGMENT} from './AssetLineageElements'; -import {AssetMaterializationMatrix, LABEL_STEP_EXECUTION_TIME} from './AssetMaterializationMatrix'; import {AssetMaterializationTable} from './AssetMaterializationTable'; import {AssetValueGraph} from './AssetValueGraph'; import {AssetKey, AssetNumericHistoricalData} from './types'; @@ -27,45 +25,39 @@ import {HistoricalMaterialization, useMaterializationBuckets} from './useMateria interface Props { assetKey: AssetKey; - asOf: string | null; asSidebarSection?: boolean; } +const LABEL_STEP_EXECUTION_TIME = 'Step Execution Time'; -export const AssetMaterializations: React.FC = ({assetKey, asOf, asSidebarSection}) => { - const before = React.useMemo(() => (asOf ? `${Number(asOf) + 1}` : ''), [asOf]); +export const AssetMaterializations: React.FC = ({assetKey, asSidebarSection}) => { const {data, loading} = useQuery( ASSET_MATERIALIZATIONS_QUERY, { variables: { assetKey: {path: assetKey.path}, limit: 200, - before, }, }, ); const asset = data?.assetOrError.__typename === 'Asset' ? data?.assetOrError : null; const assetMaterializations = asset?.assetMaterializations || []; - - const [activeTab = 'graphs', setActiveTab] = useQueryPersistedState<'graphs' | 'list'>({ - queryKey: 'tab', - }); - - const isPartitioned = assetMaterializations.some((m) => m.partition); - const [xAxis = isPartitioned ? 'partition' : 'time', setXAxis] = useQueryPersistedState< - 'partition' | 'time' - >({ - queryKey: 'axis', - }); - + const hasPartitions = assetMaterializations.some((m) => m.partition); const hasLineage = assetMaterializations.some( (m) => m.materializationEvent.assetLineage.length > 0, ); + const [{xAxis, asOf}, setParams] = useQueryPersistedState<{ + xAxis: 'partition' | 'time'; + asOf: string; + }>({ + defaults: {xAxis: hasPartitions ? 'partition' : 'time'}, + }); + const bucketed = useMaterializationBuckets({ materializations: assetMaterializations, - isPartitioned, - shouldBucketPartitions: true, + hasPartitions, + shouldBucketPartitions: xAxis === 'partition', }); const reversed = React.useMemo(() => [...bucketed].reverse(), [bucketed]); @@ -75,93 +67,82 @@ export const AssetMaterializations: React.FC = ({assetKey, asOf, asSideba return ; // chartjs and our useViewport hook don't play nicely with jest } - const content = () => { - if (loading) { - return ( - - - - ); - } - - if (!reversed.length) { - return ( - - - - ); - } - - if (activeTab === 'list') { - return ( - - ); - } + if (loading) { + return ( + + + + ); + } + if (!reversed.length) { return ( - + + + ); - }; + } if (asSidebarSection) { - return content(); + return ( + + ); } return ( -
- - Materializations over time - {isPartitioned ? ( - setXAxis(id as 'partition' | 'time')} - /> - ) : null} - - {reversed.length ? ( + + - - - - + Materializations + {hasPartitions ? ( +
+ setParams({xAxis: id as 'partition' | 'time', asOf: asOf})} + /> +
+ ) : null} - ) : null} - {content()} -
+ Number(b.latest.materializationEvent.timestamp) <= Number(asOf)) + ?.latest.materializationEvent.timestamp + } + setFocused={(asOf) => setParams({xAxis, asOf: asOf})} + /> + + + + + ); }; -const AssetMaterializationMatrixAndGraph: React.FC<{ +const AssetMaterializationGraphs: React.FC<{ assetMaterializations: HistoricalMaterialization[]; - isPartitioned: boolean; xAxis: 'partition' | 'time'; asSidebarSection?: boolean; }> = (props) => { - const {assetMaterializations, isPartitioned, xAxis} = props; + const {assetMaterializations, xAxis} = props; const [xHover, setXHover] = React.useState(null); const latest = assetMaterializations.map((m) => m.latest); @@ -172,32 +153,19 @@ const AssetMaterializationMatrixAndGraph: React.FC<{ return ( <> - {!props.asSidebarSection && ( - x !== xHover && setXHover(x)} - graphDataByMetadataLabel={graphDataByMetadataLabel} - graphedLabels={graphedLabels} - setGraphedLabels={setGraphedLabels} - /> - )}
{[...graphedLabels].sort().map((label) => ( x !== xHover && setXHover(x)} diff --git a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx new file mode 100644 index 0000000000000..373bb0bad2af5 --- /dev/null +++ b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx @@ -0,0 +1,126 @@ +import * as React from 'react'; +import {useHistory} from 'react-router-dom'; +import styled from 'styled-components/macro'; + +import {SVGViewport} from '../graph/SVGViewport'; +import {AssetLinks} from '../workspace/asset-graph/AssetLinks'; +import {AssetNode, getNodeDimensions} from '../workspace/asset-graph/AssetNode'; +import {getForeignNodeDimensions, ForeignNode} from '../workspace/asset-graph/ForeignNode'; +import { + layoutGraph, + buildGraphComputeStatuses, + GraphData, + assetKeyToString, +} from '../workspace/asset-graph/Utils'; + +import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinitionFragment'; + +const buildGraphFromSingleNode = (assetNode: AssetNodeDefinitionFragment) => { + const graphData: GraphData = { + downstream: { + [assetNode.id]: {}, + }, + nodes: { + [assetNode.id]: { + id: assetNode.id, + assetKey: assetNode.assetKey, + definition: {...assetNode, dependencies: []}, + hidden: false, + }, + }, + upstream: { + [assetNode.id]: {}, + }, + }; + + for (const {asset} of assetNode.dependencies) { + graphData.upstream[assetNode.id][asset.id] = true; + graphData.downstream[asset.id] = {...graphData.downstream[asset.id], [assetNode.id]: 'a'}; + graphData.nodes[asset.id] = { + id: asset.id, + assetKey: asset.assetKey, + definition: {...asset, dependencies: []}, + hidden: false, + }; + } + for (const {asset} of assetNode.dependedBy) { + graphData.upstream[asset.id] = {...graphData.upstream[asset.id], [assetNode.id]: true}; + graphData.downstream[assetNode.id][asset.id] = 'a'; + graphData.nodes[asset.id] = { + id: asset.id, + assetKey: asset.assetKey, + definition: {...asset, dependencies: []}, + hidden: false, + }; + } + return graphData; +}; + +export const AssetNeighborsGraph: React.FC<{assetNode: AssetNodeDefinitionFragment}> = ({ + assetNode, +}) => { + const history = useHistory(); + const graphData = buildGraphFromSingleNode(assetNode); + const layout = layoutGraph(graphData, 0); + const computeStatuses = buildGraphComputeStatuses(graphData); + + return ( + {}} + onClick={() => {}} + maxZoom={1.1} + maxAutocenterZoom={1.0} + > + {({scale: _scale}: any) => ( + + + {layout.nodes.map((layoutNode) => { + const graphNode = graphData.nodes[layoutNode.id]; + const {width, height} = graphNode.hidden + ? getForeignNodeDimensions(layoutNode.id) + : getNodeDimensions(graphNode.definition); + return ( + { + e.stopPropagation(); + history.push(`/instance/assets/${assetKeyToString(graphNode.assetKey)}`); + }} + style={{overflow: 'visible'}} + > + {graphNode.hidden ? ( + + ) : ( + + )} + + ); + })} + + )} + + ); +}; + +const SVGContainer = styled.svg` + overflow: visible; + border-radius: 0; +`; diff --git a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx index 30c725d445b8c..3a387d2227270 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx @@ -2,22 +2,79 @@ import {gql} from '@apollo/client'; import * as React from 'react'; import {Description} from '../pipelines/Description'; +import {PipelineReference} from '../pipelines/PipelineReference'; +import {Box} from '../ui/Box'; +import {ColorsWIP} from '../ui/Colors'; +import {Subheading} from '../ui/Text'; +import {ASSET_NODE_FRAGMENT} from '../workspace/asset-graph/AssetNode'; +import {AssetNeighborsGraph} from './AssetNeighborsGraph'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinitionFragment'; export const AssetNodeDefinition: React.FC<{assetNode: AssetNodeDefinitionFragment}> = ({ assetNode, }) => { return ( -
- -
+ + + + Definition in Repository + {assetNode.jobName && ( + + )} + + + + + + + + Related Assets + + + + + + ); }; export const ASSET_NODE_DEFINITION_FRAGMENT = gql` fragment AssetNodeDefinitionFragment on AssetNode { id - description + ...AssetNodeFragment + + dependencies { + asset { + id + ...AssetNodeFragment + } + } + dependedBy { + asset { + id + ...AssetNodeFragment + } + } } + ${ASSET_NODE_FRAGMENT} `; diff --git a/js_modules/dagit/packages/core/src/assets/AssetView.tsx b/js_modules/dagit/packages/core/src/assets/AssetView.tsx index 98aa44df57dfa..e2b2221e5ab97 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetView.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetView.tsx @@ -2,43 +2,28 @@ import {gql, useQuery} from '@apollo/client'; import * as React from 'react'; import {useDocumentTitle} from '../hooks/useDocumentTitle'; -import {METADATA_ENTRY_FRAGMENT} from '../runs/MetadataEntry'; import {Box} from '../ui/Box'; import {Spinner} from '../ui/Spinner'; -import {Subheading} from '../ui/Text'; import {assetKeyToString} from '../workspace/asset-graph/Utils'; import {AssetMaterializations} from './AssetMaterializations'; import {AssetNodeDefinition, ASSET_NODE_DEFINITION_FRAGMENT} from './AssetNodeDefinition'; -import { - LatestMaterializationMetadata, - LATEST_MATERIALIZATION_METADATA_FRAGMENT, -} from './LastMaterializationMetadata'; -import {SnapshotWarning, SNAPSHOT_WARNING_ASSET_FRAGMENT} from './SnapshotWarning'; import {AssetKey} from './types'; import {AssetQuery, AssetQueryVariables} from './types/AssetQuery'; interface Props { assetKey: AssetKey; - asOf: string | null; } -export const AssetView: React.FC = ({assetKey, asOf}) => { +export const AssetView: React.FC = ({assetKey}) => { useDocumentTitle(`Asset: ${assetKeyToString(assetKey)}`); - const before = React.useMemo(() => (asOf ? `${Number(asOf) + 1}` : ''), [asOf]); + const {data, loading} = useQuery(ASSET_QUERY, { variables: { assetKey: {path: assetKey.path}, - limit: 1, - before, }, }); - const isPartitioned = !!( - data?.assetOrError?.__typename === 'Asset' && - data?.assetOrError?.assetMaterializations[0]?.partition - ); - return (
@@ -48,44 +33,28 @@ export const AssetView: React.FC = ({assetKey, asOf}) => { )} - {data?.assetOrError && data.assetOrError.__typename === 'Asset' && ( + {/* {data?.assetOrError && data.assetOrError.__typename === 'Asset' && ( - )} + )} */} {data?.assetOrError && data.assetOrError.__typename === 'Asset' && data.assetOrError.definition && ( )} - - - {isPartitioned ? 'Latest materialized partition' : 'Latest materialization'} - - - {data?.assetOrError && data.assetOrError.__typename === 'Asset' && ( - - )}
- +
); }; const ASSET_QUERY = gql` - query AssetQuery($assetKey: AssetKeyInput!, $limit: Int!, $before: String) { + query AssetQuery($assetKey: AssetKeyInput!) { assetOrError(assetKey: $assetKey) { ... on Asset { id key { path } - ...SnapshotWarningAssetFragment - - assetMaterializations(limit: $limit, beforeTimestampMillis: $before) { - ...LatestMaterializationMetadataFragment - } definition { id @@ -98,8 +67,5 @@ const ASSET_QUERY = gql` } } } - ${METADATA_ENTRY_FRAGMENT} ${ASSET_NODE_DEFINITION_FRAGMENT} - ${LATEST_MATERIALIZATION_METADATA_FRAGMENT} - ${SNAPSHOT_WARNING_ASSET_FRAGMENT} `; diff --git a/js_modules/dagit/packages/core/src/assets/SnapshotWarning.tsx b/js_modules/dagit/packages/core/src/assets/SnapshotWarning.tsx deleted file mode 100644 index dbc74e967ae02..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/SnapshotWarning.tsx +++ /dev/null @@ -1,78 +0,0 @@ -import {gql} from '@apollo/client'; -import * as React from 'react'; -import {Link} from 'react-router-dom'; - -import {Timestamp} from '../app/time/Timestamp'; -import {Alert} from '../ui/Alert'; -import {Box} from '../ui/Box'; -import {ColorsWIP} from '../ui/Colors'; - -import {SnapshotWarningAssetFragment} from './types/SnapshotWarningAssetFragment'; - -export const SnapshotWarning: React.FC<{ - asset: SnapshotWarningAssetFragment; - asOf: string | null; -}> = ({asOf, asset}) => { - // If the most recent materialization and the `asOf` materialization are the same, we don't - // want to show the `Alert`. - if (!asOf) { - return null; - } - - const materializationAsOfTime = asset.assetMaterializations[0]; - const mostRecentMaterialization = asset.mostRecentMaterialization[0]; - - if ( - !materializationAsOfTime || - !mostRecentMaterialization || - materializationAsOfTime.materializationEvent.timestamp === - mostRecentMaterialization.materializationEvent.timestamp - ) { - return null; - } - - return ( - - - This view represents{' '} - {asset.key.path[asset.key.path.length - 1]} as of{' '} - - - - . You can also view the latest materialization for - this asset. - - } - /> - - ); -}; - -export const SNAPSHOT_WARNING_ASSET_FRAGMENT = gql` - fragment SnapshotWarningAssetFragment on Asset { - id - key { - path - } - mostRecentMaterialization: assetMaterializations(limit: 1) { - materializationEvent { - timestamp - } - } - assetMaterializations(limit: $limit, beforeTimestampMillis: $before) { - materializationEvent { - timestamp - } - } - } -`; diff --git a/js_modules/dagit/packages/core/src/assets/Sparkline.tsx b/js_modules/dagit/packages/core/src/assets/Sparkline.tsx deleted file mode 100644 index 96d909bd553f6..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/Sparkline.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import React from 'react'; - -import {ColorsWIP} from '../ui/Colors'; - -import {AssetNumericHistoricalData} from './types'; - -export const Sparkline: React.FunctionComponent<{ - data: AssetNumericHistoricalData[0]; - width: number; - height: number; -}> = ({data, width, height}) => { - const ref = React.createRef(); - React.useEffect(() => { - if (!ref.current) { - return; - } - const {width, height} = ref.current; - const ctx = ref.current.getContext('2d'); - if (!ctx) { - return; - } - const margin = height * 0.15; - const yScale = (height - margin * 2) / (data.maxY - data.minY); - const xScale = width / (data.maxXNumeric - data.minXNumeric); - - ctx.clearRect(0, 0, width, height); - ctx.resetTransform(); - ctx.scale(1, -1); - ctx.translate(0, -height); - - ctx.beginPath(); - ctx.strokeStyle = ColorsWIP.Blue500; - ctx.lineWidth = 3; - ctx.lineJoin = 'round'; - ctx.lineCap = 'round'; - let penUp = true; - for (let ii = 0; ii < data.values.length; ii++) { - const v = data.values[ii]; - if (v.y === null) { - penUp = true; - } else { - const xPx = xScale * (v.xNumeric - data.minXNumeric); - const yPx = margin + 0.5 + Math.round(yScale * (v.y - data.minY)); - if (penUp) { - ctx.moveTo(xPx - 2, yPx); - ctx.lineTo(xPx, yPx); - penUp = false; - } else { - ctx.lineTo(xPx, yPx); - } - } - } - ctx.stroke(); - }, [ref, data]); - - // Note: canvas `width` attribute is @2x the CSS width to force retina rendering on all displays - return ( - - ); -}; diff --git a/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionFragment.ts b/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionFragment.ts index 023ea24ffe0df..883c26e046e84 100644 --- a/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionFragment.ts +++ b/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionFragment.ts @@ -3,12 +3,494 @@ // @generated // This file was automatically generated and should not be edited. +import { RunStatus } from "./../../types/globalTypes"; + // ==================================================== // GraphQL fragment: AssetNodeDefinitionFragment // ==================================================== +export interface AssetNodeDefinitionFragment_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetNodeDefinitionFragment_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetNodeDefinitionFragment_assetMaterializations_runOrError = AssetNodeDefinitionFragment_assetMaterializations_runOrError_RunNotFoundError | AssetNodeDefinitionFragment_assetMaterializations_runOrError_Run; + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_stepStats; + materialization: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetNodeDefinitionFragment_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetNodeDefinitionFragment_assetMaterializations_runOrError; + materializationEvent: AssetNodeDefinitionFragment_assetMaterializations_materializationEvent; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError = AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError_RunNotFoundError | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError_Run; + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_stepStats; + materialization: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_runOrError; + materializationEvent: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations_materializationEvent; +} + +export interface AssetNodeDefinitionFragment_dependencies_asset { + __typename: "AssetNode"; + id: string; + opName: string | null; + description: string | null; + jobName: string | null; + assetKey: AssetNodeDefinitionFragment_dependencies_asset_assetKey; + assetMaterializations: AssetNodeDefinitionFragment_dependencies_asset_assetMaterializations[]; +} + +export interface AssetNodeDefinitionFragment_dependencies { + __typename: "AssetDependency"; + asset: AssetNodeDefinitionFragment_dependencies_asset; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError = AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError_RunNotFoundError | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError_Run; + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_stepStats; + materialization: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_runOrError; + materializationEvent: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations_materializationEvent; +} + +export interface AssetNodeDefinitionFragment_dependedBy_asset { + __typename: "AssetNode"; + id: string; + opName: string | null; + description: string | null; + jobName: string | null; + assetKey: AssetNodeDefinitionFragment_dependedBy_asset_assetKey; + assetMaterializations: AssetNodeDefinitionFragment_dependedBy_asset_assetMaterializations[]; +} + +export interface AssetNodeDefinitionFragment_dependedBy { + __typename: "AssetDependency"; + asset: AssetNodeDefinitionFragment_dependedBy_asset; +} + export interface AssetNodeDefinitionFragment { __typename: "AssetNode"; id: string; + opName: string | null; description: string | null; + jobName: string | null; + assetKey: AssetNodeDefinitionFragment_assetKey; + assetMaterializations: AssetNodeDefinitionFragment_assetMaterializations[]; + dependencies: AssetNodeDefinitionFragment_dependencies[]; + dependedBy: AssetNodeDefinitionFragment_dependedBy[]; } diff --git a/js_modules/dagit/packages/core/src/assets/types/AssetQuery.ts b/js_modules/dagit/packages/core/src/assets/types/AssetQuery.ts index ddc102a1ada78..292cff84a05c7 100644 --- a/js_modules/dagit/packages/core/src/assets/types/AssetQuery.ts +++ b/js_modules/dagit/packages/core/src/assets/types/AssetQuery.ts @@ -3,7 +3,7 @@ // @generated // This file was automatically generated and should not be edited. -import { AssetKeyInput } from "./../../types/globalTypes"; +import { AssetKeyInput, RunStatus } from "./../../types/globalTypes"; // ==================================================== // GraphQL query operation: AssetQuery @@ -18,58 +18,226 @@ export interface AssetQuery_assetOrError_Asset_key { path: string[]; } -export interface AssetQuery_assetOrError_Asset_mostRecentMaterialization_materializationEvent { +export interface AssetQuery_assetOrError_Asset_definition_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError = AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError_RunNotFoundError | AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError_Run; + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent { __typename: "StepMaterializationEvent"; + runId: string; timestamp: string; + stepKey: string | null; + stepStats: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_stepStats; + materialization: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent_assetLineage[]; } -export interface AssetQuery_assetOrError_Asset_mostRecentMaterialization { +export interface AssetQuery_assetOrError_Asset_definition_assetMaterializations { __typename: "AssetMaterialization"; - materializationEvent: AssetQuery_assetOrError_Asset_mostRecentMaterialization_materializationEvent; + partition: string | null; + runOrError: AssetQuery_assetOrError_Asset_definition_assetMaterializations_runOrError; + materializationEvent: AssetQuery_assetOrError_Asset_definition_assetMaterializations_materializationEvent; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_stepStats { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError = AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError_RunNotFoundError | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError_Run; + +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_stepStats { __typename: "RunStepStats"; endTime: number | null; startTime: number | null; + stepKey: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { __typename: "EventPathMetadataEntry"; label: string; description: string | null; path: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { __typename: "EventJsonMetadataEntry"; label: string; description: string | null; jsonString: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { __typename: "EventUrlMetadataEntry"; label: string; description: string | null; url: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { __typename: "EventTextMetadataEntry"; label: string; description: string | null; text: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { __typename: "EventMarkdownMetadataEntry"; label: string; description: string | null; mdStr: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { __typename: "EventPythonArtifactMetadataEntry"; label: string; description: string | null; @@ -77,14 +245,14 @@ export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializa name: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { __typename: "EventFloatMetadataEntry"; label: string; description: string | null; floatValue: number | null; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { __typename: "EventIntMetadataEntry"; label: string; description: string | null; @@ -92,81 +260,236 @@ export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializa intRepr: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { __typename: "EventPipelineRunMetadataEntry"; label: string; description: string | null; runId: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { __typename: "AssetKey"; path: string[]; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { __typename: "EventAssetMetadataEntry"; label: string; description: string | null; - assetKey: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; + assetKey: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; } -export type AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; +export type AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization { __typename: "Materialization"; - metadataEntries: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization_metadataEntries[]; + metadataEntries: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization_metadataEntries[]; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage_assetKey { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_assetLineage_assetKey { __typename: "AssetKey"; path: string[]; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_assetLineage { __typename: "AssetLineageInfo"; - assetKey: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage_assetKey; + assetKey: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_assetLineage_assetKey; partitions: string[]; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent { __typename: "StepMaterializationEvent"; - timestamp: string; runId: string; + timestamp: string; stepKey: string | null; - stepStats: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_stepStats; - materialization: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_materialization; - assetLineage: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage[]; + stepStats: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_stepStats; + materialization: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent_assetLineage[]; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_runOrError_RunNotFoundError { +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_runOrError; + materializationEvent: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations_materializationEvent; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependencies_asset { + __typename: "AssetNode"; + id: string; + opName: string | null; + description: string | null; + jobName: string | null; + assetKey: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetKey; + assetMaterializations: AssetQuery_assetOrError_Asset_definition_dependencies_asset_assetMaterializations[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependencies { + __typename: "AssetDependency"; + asset: AssetQuery_assetOrError_Asset_definition_dependencies_asset; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError_RunNotFoundError { __typename: "RunNotFoundError" | "PythonError"; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_runOrError_Run_repositoryOrigin { +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError_Run_repositoryOrigin { __typename: "RepositoryOrigin"; id: string; repositoryName: string; repositoryLocationName: string; } -export interface AssetQuery_assetOrError_Asset_assetMaterializations_runOrError_Run { +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError_Run { __typename: "Run"; id: string; runId: string; mode: string; pipelineName: string; pipelineSnapshotId: string | null; - repositoryOrigin: AssetQuery_assetOrError_Asset_assetMaterializations_runOrError_Run_repositoryOrigin | null; + repositoryOrigin: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError = AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError_RunNotFoundError | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError_Run; + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; } -export type AssetQuery_assetOrError_Asset_assetMaterializations_runOrError = AssetQuery_assetOrError_Asset_assetMaterializations_runOrError_RunNotFoundError | AssetQuery_assetOrError_Asset_assetMaterializations_runOrError_Run; +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} -export interface AssetQuery_assetOrError_Asset_assetMaterializations { +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_stepStats; + materialization: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations { __typename: "AssetMaterialization"; - materializationEvent: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent; partition: string | null; - runOrError: AssetQuery_assetOrError_Asset_assetMaterializations_runOrError; + runOrError: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_runOrError; + materializationEvent: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations_materializationEvent; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy_asset { + __typename: "AssetNode"; + id: string; + opName: string | null; + description: string | null; + jobName: string | null; + assetKey: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetKey; + assetMaterializations: AssetQuery_assetOrError_Asset_definition_dependedBy_asset_assetMaterializations[]; +} + +export interface AssetQuery_assetOrError_Asset_definition_dependedBy { + __typename: "AssetDependency"; + asset: AssetQuery_assetOrError_Asset_definition_dependedBy_asset; } export interface AssetQuery_assetOrError_Asset_definition { @@ -175,14 +498,16 @@ export interface AssetQuery_assetOrError_Asset_definition { description: string | null; opName: string | null; jobName: string | null; + assetKey: AssetQuery_assetOrError_Asset_definition_assetKey; + assetMaterializations: AssetQuery_assetOrError_Asset_definition_assetMaterializations[]; + dependencies: AssetQuery_assetOrError_Asset_definition_dependencies[]; + dependedBy: AssetQuery_assetOrError_Asset_definition_dependedBy[]; } export interface AssetQuery_assetOrError_Asset { __typename: "Asset"; id: string; key: AssetQuery_assetOrError_Asset_key; - mostRecentMaterialization: AssetQuery_assetOrError_Asset_mostRecentMaterialization[]; - assetMaterializations: AssetQuery_assetOrError_Asset_assetMaterializations[]; definition: AssetQuery_assetOrError_Asset_definition | null; } @@ -194,6 +519,4 @@ export interface AssetQuery { export interface AssetQueryVariables { assetKey: AssetKeyInput; - limit: number; - before?: string | null; } diff --git a/js_modules/dagit/packages/core/src/assets/types/SnapshotWarningAssetFragment.ts b/js_modules/dagit/packages/core/src/assets/types/SnapshotWarningAssetFragment.ts deleted file mode 100644 index 985b387cb07e2..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/types/SnapshotWarningAssetFragment.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// @generated -// This file was automatically generated and should not be edited. - -// ==================================================== -// GraphQL fragment: SnapshotWarningAssetFragment -// ==================================================== - -export interface SnapshotWarningAssetFragment_key { - __typename: "AssetKey"; - path: string[]; -} - -export interface SnapshotWarningAssetFragment_mostRecentMaterialization_materializationEvent { - __typename: "StepMaterializationEvent"; - timestamp: string; -} - -export interface SnapshotWarningAssetFragment_mostRecentMaterialization { - __typename: "AssetMaterialization"; - materializationEvent: SnapshotWarningAssetFragment_mostRecentMaterialization_materializationEvent; -} - -export interface SnapshotWarningAssetFragment_assetMaterializations_materializationEvent { - __typename: "StepMaterializationEvent"; - timestamp: string; -} - -export interface SnapshotWarningAssetFragment_assetMaterializations { - __typename: "AssetMaterialization"; - materializationEvent: SnapshotWarningAssetFragment_assetMaterializations_materializationEvent; -} - -export interface SnapshotWarningAssetFragment { - __typename: "Asset"; - id: string; - key: SnapshotWarningAssetFragment_key; - mostRecentMaterialization: SnapshotWarningAssetFragment_mostRecentMaterialization[]; - assetMaterializations: SnapshotWarningAssetFragment_assetMaterializations[]; -} diff --git a/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx b/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx index 3146d73ab51f3..589c6c8c6e07d 100644 --- a/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx +++ b/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx @@ -11,7 +11,7 @@ export type HistoricalMaterialization = { type Config = { materializations: AssetMaterializationFragment[]; - isPartitioned: boolean; + hasPartitions: boolean; shouldBucketPartitions: boolean; }; @@ -20,9 +20,9 @@ type Config = { * materialization separated from predecessor materializations. */ export const useMaterializationBuckets = (config: Config): HistoricalMaterialization[] => { - const {isPartitioned, materializations, shouldBucketPartitions} = config; + const {hasPartitions, materializations, shouldBucketPartitions} = config; return React.useMemo(() => { - if (!isPartitioned || !shouldBucketPartitions) { + if (!hasPartitions || !shouldBucketPartitions) { return materializations.map((materialization) => ({ latest: materialization, })); @@ -53,5 +53,5 @@ export const useMaterializationBuckets = (config: Config): HistoricalMaterializa .filter((key) => key !== NO_PARTITION_KEY) .map(separate) .concat(buckets.hasOwnProperty(NO_PARTITION_KEY) ? [separate(NO_PARTITION_KEY)] : []); - }, [isPartitioned, materializations, shouldBucketPartitions]); + }, [hasPartitions, materializations, shouldBucketPartitions]); }; diff --git a/js_modules/dagit/packages/core/src/graphql/schema.graphql b/js_modules/dagit/packages/core/src/graphql/schema.graphql index a48f6b919352a..feec504481d6f 100644 --- a/js_modules/dagit/packages/core/src/graphql/schema.graphql +++ b/js_modules/dagit/packages/core/src/graphql/schema.graphql @@ -617,12 +617,13 @@ type AssetNode { opName: String jobName: String dependencies: [AssetDependency!]! + dependedBy: [AssetDependency!]! assetMaterializations(partitions: [String], beforeTimestampMillis: String, limit: Int): [AssetMaterialization!]! } type AssetDependency { inputName: String! - upstreamAsset: AssetNode! + asset: AssetNode! } union DagsterRunEvent = ExecutionStepFailureEvent | ExecutionStepInputEvent | ExecutionStepOutputEvent | ExecutionStepSkippedEvent | ExecutionStepStartEvent | ExecutionStepSuccessEvent | ExecutionStepUpForRetryEvent | ExecutionStepRestartEvent | LogMessageEvent | RunFailureEvent | RunStartEvent | RunEnqueuedEvent | RunDequeuedEvent | RunStartingEvent | RunCancelingEvent | RunCanceledEvent | RunSuccessEvent | HandledOutputEvent | LoadedInputEvent | LogsCapturedEvent | ObjectStoreOperationEvent | StepExpectationResultEvent | StepMaterializationEvent | EngineEvent | HookCompletedEvent | HookSkippedEvent | HookErroredEvent | AlertStartEvent | AlertSuccessEvent diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index dfcb00af0c695..f03679d93287f 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -4,7 +4,6 @@ import React from 'react'; import styled from 'styled-components/macro'; import {filterByQuery} from '../../app/GraphQueryImpl'; -import {LATEST_MATERIALIZATION_METADATA_FRAGMENT} from '../../assets/LastMaterializationMetadata'; import {LaunchRootExecutionButton} from '../../execute/LaunchRootExecutionButton'; import {SVGViewport} from '../../graph/SVGViewport'; import {useDocumentTitle} from '../../hooks/useDocumentTitle'; @@ -308,7 +307,7 @@ const ASSETS_GRAPH_QUERY = gql` } dependencies { inputName - upstreamAsset { + asset { id assetKey { path @@ -328,7 +327,6 @@ const ASSETS_GRAPH_QUERY = gql` } } ${ASSET_NODE_FRAGMENT} - ${LATEST_MATERIALIZATION_METADATA_FRAGMENT} `; const SVGContainer = styled.svg` diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx index ac9d511eea7ea..8182b5bb712da 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/ForeignNode.tsx @@ -1,3 +1,4 @@ +import {isEqual} from 'lodash'; import React from 'react'; import styled from 'styled-components/macro'; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx index 8688f0baf4fd0..467710889a091 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx @@ -50,8 +50,8 @@ export const buildGraphData = (repository: Repository, jobName?: string) => { repository.assetNodes.forEach((definition: AssetNode) => { const assetKeyJson = JSON.stringify(definition.assetKey.path); - definition.dependencies.forEach(({upstreamAsset, inputName}) => { - const upstreamAssetKeyJson = JSON.stringify(upstreamAsset.assetKey.path); + definition.dependencies.forEach(({asset, inputName}) => { + const upstreamAssetKeyJson = JSON.stringify(asset.assetKey.path); downstream[upstreamAssetKeyJson] = { ...(downstream[upstreamAssetKeyJson] || {}), [assetKeyJson]: inputName, diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts index 4fa84adc101ad..5e00b0506170a 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts @@ -167,21 +167,21 @@ export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetMa materializationEvent: AssetGraphQuery_repositoryOrError_Repository_assetNodes_assetMaterializations_materializationEvent; } -export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset_assetKey { +export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_asset_assetKey { __typename: "AssetKey"; path: string[]; } -export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset { +export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_asset { __typename: "AssetNode"; id: string; - assetKey: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset_assetKey; + assetKey: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_asset_assetKey; } export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies { __typename: "AssetDependency"; inputName: string; - upstreamAsset: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_upstreamAsset; + asset: AssetGraphQuery_repositoryOrError_Repository_assetNodes_dependencies_asset; } export interface AssetGraphQuery_repositoryOrError_Repository_assetNodes { diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py b/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py index 64291cabbe658..cb6d2d37f37c7 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py @@ -14,21 +14,21 @@ class Meta: name = "AssetDependency" inputName = graphene.NonNull(graphene.String) - upstreamAsset = graphene.NonNull("dagster_graphql.schema.asset_graph.GrapheneAssetNode") + asset = graphene.NonNull("dagster_graphql.schema.asset_graph.GrapheneAssetNode") - def __init__(self, external_repository, input_name, upstream_asset_key): + def __init__(self, external_repository, input_name, asset_key): self._external_repository = check.inst_param( external_repository, "external_repository", ExternalRepository ) - self._upstream_asset_key = check.inst_param( - upstream_asset_key, "upstream_asset_key", AssetKey + self._asset_key = check.inst_param( + asset_key, "asset_key", AssetKey ) super().__init__(inputName=input_name) - def resolve_upstreamAsset(self, _graphene_info): + def resolve_asset(self, _graphene_info): return GrapheneAssetNode( self._external_repository, - self._external_repository.get_external_asset_node(self._upstream_asset_key), + self._external_repository.get_external_asset_node(self._asset_key), ) @@ -39,6 +39,7 @@ class GrapheneAssetNode(graphene.ObjectType): opName = graphene.String() jobName = graphene.String() dependencies = non_null_list(GrapheneAssetDependency) + dependedBy = non_null_list(GrapheneAssetDependency) assetMaterializations = graphene.Field( non_null_list(GrapheneAssetMaterialization), partitions=graphene.List(graphene.String), @@ -69,11 +70,24 @@ def resolve_dependencies(self, _graphene_info): GrapheneAssetDependency( external_repository=self._external_repository, input_name=dep.input_name, - upstream_asset_key=dep.upstream_asset_key, + asset_key=dep.upstream_asset_key, ) for dep in self._external_asset_node.dependencies ] + def resolve_dependedBy(self, _graphene_info): + results = [] + for item in self._external_repository.get_external_asset_nodes(): + for dep in item.dependencies: + if dep.upstream_asset_key == self._external_asset_node.asset_key: + results.append(GrapheneAssetDependency( + external_repository=self._external_repository, + input_name=dep.input_name, + asset_key=item.asset_key, + )) + return results + + def resolve_assetMaterializations(self, graphene_info, **kwargs): from ..implementation.fetch_assets import get_asset_events From d9258c06831841e43316ffb1f49daef12308309e Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 23 Nov 2021 09:23:41 -0600 Subject: [PATCH 05/22] Fix tests --- .../dagit/packages/core/src/assets/AssetNeighborsGraph.tsx | 4 ++-- .../src/workspace/asset-graph/useLaunchSingleAssetJob.tsx | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx index 373bb0bad2af5..2fab52fbd4133 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx @@ -39,7 +39,7 @@ const buildGraphFromSingleNode = (assetNode: AssetNodeDefinitionFragment) => { graphData.nodes[asset.id] = { id: asset.id, assetKey: asset.assetKey, - definition: {...asset, dependencies: []}, + definition: {...asset, dependencies: [], dependedBy: []}, hidden: false, }; } @@ -49,7 +49,7 @@ const buildGraphFromSingleNode = (assetNode: AssetNodeDefinitionFragment) => { graphData.nodes[asset.id] = { id: asset.id, assetKey: asset.assetKey, - definition: {...asset, dependencies: []}, + definition: {...asset, dependencies: [], dependedBy: []}, hidden: false, }; } diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/useLaunchSingleAssetJob.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/useLaunchSingleAssetJob.tsx index 2e1429ff616cf..44af46cb3c3cd 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/useLaunchSingleAssetJob.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/useLaunchSingleAssetJob.tsx @@ -15,7 +15,10 @@ export const useLaunchSingleAssetJob = () => { ); return React.useCallback( - async (repoAddress: RepoAddress, definition: {jobName: string | null; opName: string}) => { + async ( + repoAddress: RepoAddress, + definition: {jobName: string | null; opName: string | null}, + ) => { if (!definition.jobName) { return; } From 1f985f4bd549277ee2e3dca8d7b510e7dd21ca4a Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 23 Nov 2021 10:41:55 -0600 Subject: [PATCH 06/22] Replace neighbors graph with neighbors list --- .../core/src/assets/AssetNeighborsGraph.tsx | 126 ------------------ .../core/src/assets/AssetNodeDefinition.tsx | 85 +++++++++--- .../packages/core/src/assets/AssetView.tsx | 5 +- .../src/nav/LeftNavRepositorySection.test.tsx | 1 + .../core/src/pipelines/PipelineRoot.test.tsx | 1 + .../packages/core/src/testing/defaultMocks.ts | 1 + 6 files changed, 73 insertions(+), 146 deletions(-) delete mode 100644 js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx diff --git a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx deleted file mode 100644 index 2fab52fbd4133..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx +++ /dev/null @@ -1,126 +0,0 @@ -import * as React from 'react'; -import {useHistory} from 'react-router-dom'; -import styled from 'styled-components/macro'; - -import {SVGViewport} from '../graph/SVGViewport'; -import {AssetLinks} from '../workspace/asset-graph/AssetLinks'; -import {AssetNode, getNodeDimensions} from '../workspace/asset-graph/AssetNode'; -import {getForeignNodeDimensions, ForeignNode} from '../workspace/asset-graph/ForeignNode'; -import { - layoutGraph, - buildGraphComputeStatuses, - GraphData, - assetKeyToString, -} from '../workspace/asset-graph/Utils'; - -import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinitionFragment'; - -const buildGraphFromSingleNode = (assetNode: AssetNodeDefinitionFragment) => { - const graphData: GraphData = { - downstream: { - [assetNode.id]: {}, - }, - nodes: { - [assetNode.id]: { - id: assetNode.id, - assetKey: assetNode.assetKey, - definition: {...assetNode, dependencies: []}, - hidden: false, - }, - }, - upstream: { - [assetNode.id]: {}, - }, - }; - - for (const {asset} of assetNode.dependencies) { - graphData.upstream[assetNode.id][asset.id] = true; - graphData.downstream[asset.id] = {...graphData.downstream[asset.id], [assetNode.id]: 'a'}; - graphData.nodes[asset.id] = { - id: asset.id, - assetKey: asset.assetKey, - definition: {...asset, dependencies: [], dependedBy: []}, - hidden: false, - }; - } - for (const {asset} of assetNode.dependedBy) { - graphData.upstream[asset.id] = {...graphData.upstream[asset.id], [assetNode.id]: true}; - graphData.downstream[assetNode.id][asset.id] = 'a'; - graphData.nodes[asset.id] = { - id: asset.id, - assetKey: asset.assetKey, - definition: {...asset, dependencies: [], dependedBy: []}, - hidden: false, - }; - } - return graphData; -}; - -export const AssetNeighborsGraph: React.FC<{assetNode: AssetNodeDefinitionFragment}> = ({ - assetNode, -}) => { - const history = useHistory(); - const graphData = buildGraphFromSingleNode(assetNode); - const layout = layoutGraph(graphData, 0); - const computeStatuses = buildGraphComputeStatuses(graphData); - - return ( - {}} - onClick={() => {}} - maxZoom={1.1} - maxAutocenterZoom={1.0} - > - {({scale: _scale}: any) => ( - - - {layout.nodes.map((layoutNode) => { - const graphNode = graphData.nodes[layoutNode.id]; - const {width, height} = graphNode.hidden - ? getForeignNodeDimensions(layoutNode.id) - : getNodeDimensions(graphNode.definition); - return ( - { - e.stopPropagation(); - history.push(`/instance/assets/${assetKeyToString(graphNode.assetKey)}`); - }} - style={{overflow: 'visible'}} - > - {graphNode.hidden ? ( - - ) : ( - - )} - - ); - })} - - )} - - ); -}; - -const SVGContainer = styled.svg` - overflow: visible; - border-radius: 0; -`; diff --git a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx index 3a387d2227270..6fa6086c02fdd 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx @@ -1,14 +1,20 @@ import {gql} from '@apollo/client'; import * as React from 'react'; +import {useHistory} from 'react-router-dom'; import {Description} from '../pipelines/Description'; import {PipelineReference} from '../pipelines/PipelineReference'; import {Box} from '../ui/Box'; import {ColorsWIP} from '../ui/Colors'; import {Subheading} from '../ui/Text'; -import {ASSET_NODE_FRAGMENT} from '../workspace/asset-graph/AssetNode'; +import { + AssetNode, + ASSET_NODE_FRAGMENT, + getNodeDimensions, +} from '../workspace/asset-graph/AssetNode'; +import {assetKeyToString} from '../workspace/asset-graph/Utils'; +import {AssetNodeFragment} from '../workspace/asset-graph/types/AssetNodeFragment'; -import {AssetNeighborsGraph} from './AssetNeighborsGraph'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinitionFragment'; export const AssetNodeDefinition: React.FC<{assetNode: AssetNodeDefinitionFragment}> = ({ @@ -19,7 +25,23 @@ export const AssetNodeDefinition: React.FC<{assetNode: AssetNodeDefinitionFragme flex={{direction: 'row'}} border={{side: 'bottom', width: 4, color: ColorsWIP.KeylineGray}} > - + + + Parent Assets ({assetNode.dependencies.length}) + + + + Child Assets ({assetNode.dependedBy.length}) + + + + )} - + - - - Related Assets - - - - - ); }; @@ -78,3 +85,43 @@ export const ASSET_NODE_DEFINITION_FRAGMENT = gql` } ${ASSET_NODE_FRAGMENT} `; + +const AssetList: React.FC<{ + items: {asset: AssetNodeFragment}[]; +}> = ({items}) => { + const history = useHistory(); + + return ( + + {items.map((dep) => ( +
{ + if (e.isDefaultPrevented()) { + return; + } + history.push(`/instance/assets/${assetKeyToString(dep.asset.assetKey)}`); + }} + > + +
+ ))} + + ); +}; diff --git a/js_modules/dagit/packages/core/src/assets/AssetView.tsx b/js_modules/dagit/packages/core/src/assets/AssetView.tsx index e2b2221e5ab97..e16c8354ae777 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetView.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetView.tsx @@ -28,7 +28,10 @@ export const AssetView: React.FC = ({assetKey}) => {
{loading && ( - + )} diff --git a/js_modules/dagit/packages/core/src/nav/LeftNavRepositorySection.test.tsx b/js_modules/dagit/packages/core/src/nav/LeftNavRepositorySection.test.tsx index bca6adc4d3dec..43dc6dfcf73b7 100644 --- a/js_modules/dagit/packages/core/src/nav/LeftNavRepositorySection.test.tsx +++ b/js_modules/dagit/packages/core/src/nav/LeftNavRepositorySection.test.tsx @@ -34,6 +34,7 @@ describe('Repository options', () => { id: () => 'my_pipeline', name: () => 'my_pipeline', modes: () => new MockList(1), + isAssetJob: () => false, }), }; diff --git a/js_modules/dagit/packages/core/src/pipelines/PipelineRoot.test.tsx b/js_modules/dagit/packages/core/src/pipelines/PipelineRoot.test.tsx index 1f72d11f14d42..138fc1d5ac44f 100644 --- a/js_modules/dagit/packages/core/src/pipelines/PipelineRoot.test.tsx +++ b/js_modules/dagit/packages/core/src/pipelines/PipelineRoot.test.tsx @@ -24,6 +24,7 @@ describe('PipelineRoot', () => { Pipeline: () => ({ id: () => PIPELINE_NAME, modes: () => new MockList(1), + isAssetJob: () => false, }), PipelineSnapshot: () => ({ runs: () => new MockList(0), diff --git a/js_modules/dagit/packages/core/src/testing/defaultMocks.ts b/js_modules/dagit/packages/core/src/testing/defaultMocks.ts index e160e69e3996f..3ddfb9bd3ca64 100644 --- a/js_modules/dagit/packages/core/src/testing/defaultMocks.ts +++ b/js_modules/dagit/packages/core/src/testing/defaultMocks.ts @@ -24,6 +24,7 @@ export const defaultMocks = { Pipeline: () => ({ id: randomId, isJob: () => false, + isAssetJob: () => false, name: hyphenatedName, pipelineSnapshotId: randomId, schedules: () => new MockList(0), From 5b41c15d3e642f06be845996191a2e15195c684a Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 23 Nov 2021 11:43:04 -0600 Subject: [PATCH 07/22] Fix asOf handling, add disclosure triangles + hover states --- .../src/assets/AssetMaterializationTable.tsx | 58 +++++++++++----- .../core/src/assets/AssetMaterializations.tsx | 51 +++++++------- .../packages/core/src/assets/AssetView.tsx | 68 +++++++++++++++---- .../src/assets/useMaterializationBuckets.tsx | 4 +- .../asset-graph/AssetGraphExplorer.tsx | 9 ++- .../src/workspace/asset-graph/AssetNode.tsx | 20 ++++-- .../asset-graph/SidebarAssetInfo.tsx | 8 ++- 7 files changed, 157 insertions(+), 61 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx index 3c9b6ffacee7e..dd00fda4157d0 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx @@ -80,27 +80,33 @@ const AssetMaterializationRow: React.FC<{ const isFocused = focused === timestamp; const focusCss = isFocused - ? {borderLeft: `5px solid ${ColorsWIP.HighlightGreen}`} - : {paddingLeft: 29}; + ? {paddingLeft: 4, borderLeft: `4px solid ${ColorsWIP.HighlightGreen}`} + : {paddingLeft: 8}; return ( <> -
setFocused?.(timestamp)}> + setFocused?.(timestamp)}> {hasPartitions && ( - )} - + {isFocused && ( - - - - ))} - {hasLineage && ( - - - - - )} + + {(metadataEntries || []).map((entry) => ( + + + + + ))} + {hasLineage && ( + + + + + )} + ) : ( No materialization event metadata diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index 3198c47ee7057..514e42c425062 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -10,6 +10,8 @@ import {ColorsWIP} from '../ui/Colors'; import {NonIdealState} from '../ui/NonIdealState'; import {Spinner} from '../ui/Spinner'; import {Subheading} from '../ui/Text'; +import {InProgressRunsBanner} from '../workspace/asset-graph/InProgressRunsBanner'; +import {InProgressRunsFragment} from '../workspace/asset-graph/types/InProgressRunsFragment'; import {ASSET_LINEAGE_FRAGMENT} from './AssetLineageElements'; import {AssetMaterializationTable} from './AssetMaterializationTable'; @@ -26,6 +28,7 @@ import {HistoricalMaterialization, useMaterializationBuckets} from './useMateria interface Props { assetKey: AssetKey; asSidebarSection?: boolean; + inProgressRunIds?: string[]; params: AssetViewParams; paramsTimeWindowOnly: boolean; setParams: (params: AssetViewParams) => void; @@ -38,6 +41,7 @@ export const AssetMaterializations: React.FC = ({ params, setParams, paramsTimeWindowOnly, + inProgressRunIds, }) => { const {data, loading} = useQuery( ASSET_MATERIALIZATIONS_QUERY, @@ -117,6 +121,7 @@ export const AssetMaterializations: React.FC = ({ ) : null} + {inProgressRunIds && } { return graphData; }; -export const AssetNeighborsGraph: React.FC<{assetNode: AssetNodeDefinitionFragment}> = ({ - assetNode, -}) => { +export const AssetNeighborsGraph: React.FC<{ + assetNode: AssetNodeDefinitionFragment; + repoAddress: RepoAddress; + liveDataByNode: LiveData; +}> = ({assetNode, liveDataByNode, repoAddress}) => { const history = useHistory(); const graphData = buildGraphFromSingleNode(assetNode); const layout = layoutGraph(graphData, {margin: 0, mini: true}); - const computeStatuses = buildGraphComputeStatuses(graphData); return ( )} diff --git a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx index 2d7a575fb99bd..a939c1ee3f79b 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx @@ -1,68 +1,27 @@ -import {gql, useQuery} from '@apollo/client'; +import {gql} from '@apollo/client'; import * as React from 'react'; -import {useHistory} from 'react-router-dom'; import {Description} from '../pipelines/Description'; import {PipelineReference} from '../pipelines/PipelineReference'; import {Box} from '../ui/Box'; import {ColorsWIP} from '../ui/Colors'; import {Subheading} from '../ui/Text'; -<<<<<<< HEAD import {useRepositoryOptions} from '../workspace/WorkspaceContext'; -import { - AssetNode, - ASSET_NODE_FRAGMENT, - ASSET_NODE_LIVE_FRAGMENT, -} from '../workspace/asset-graph/AssetNode'; -import { - assetKeyToString, - buildGraphDataFromSingleNode, - buildLiveData, - LiveData, -} from '../workspace/asset-graph/Utils'; -import {AssetNodeFragment} from '../workspace/asset-graph/types/AssetNodeFragment'; +import {ASSET_NODE_FRAGMENT, ASSET_NODE_LIVE_FRAGMENT} from '../workspace/asset-graph/AssetNode'; +import {buildGraphDataFromSingleNode, buildLiveData} from '../workspace/asset-graph/Utils'; +import {InProgressRunsFragment} from '../workspace/asset-graph/types/InProgressRunsFragment'; import {findRepoContainingPipeline} from '../workspace/findRepoContainingPipeline'; -======= -import {ASSET_NODE_FRAGMENT} from '../workspace/asset-graph/AssetNode'; ->>>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 import {AssetNeighborsGraph} from './AssetNeighborsGraph'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinitionFragment'; -import { - AssetNodeDefinitionRunsQuery, - AssetNodeDefinitionRunsQueryVariables, -} from './types/AssetNodeDefinitionRunsQuery'; -export const AssetNodeDefinition: React.FC<{assetNode: AssetNodeDefinitionFragment}> = ({ - assetNode, -}) => { +export const AssetNodeDefinition: React.FC<{ + assetNode: AssetNodeDefinitionFragment; + inProgressRuns: InProgressRunsFragment[]; +}> = ({assetNode, inProgressRuns}) => { const {options} = useRepositoryOptions(); - const repos = assetNode.jobName ? findRepoContainingPipeline(options, assetNode.jobName) : []; + const [repo] = assetNode?.jobName ? findRepoContainingPipeline(options, assetNode.jobName) : []; - const liveResult = useQuery( - ASSET_NODE_DEFINITION_RUNS_QUERY, - { - skip: !assetNode.jobName || !repos.length, - variables: { - pipelineSelector: { - pipelineName: assetNode.jobName || '', - repositoryLocationName: repos[0].repositoryLocation.name, - repositoryName: repos[0].repository.name, - }, - repositorySelector: { - repositoryLocationName: repos[0].repositoryLocation.name, - repositoryName: repos[0].repository.name, - }, - }, - notifyOnNetworkStatusChange: true, - pollInterval: 5 * 1000, - }, - ); - - const inProgress = - liveResult.data?.repositoryOrError.__typename === 'Repository' - ? liveResult.data.repositoryOrError.inProgressRunsByStep - : []; const nodesWithLatestMaterialization = [ assetNode, ...assetNode.dependencies.map((d) => d.asset), @@ -71,8 +30,9 @@ export const AssetNodeDefinition: React.FC<{assetNode: AssetNodeDefinitionFragme const liveDataByNode = buildLiveData( buildGraphDataFromSingleNode(assetNode), nodesWithLatestMaterialization, - inProgress, + inProgressRuns, ); + console.log(liveDataByNode); return ( >>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 /> >>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 > -<<<<<<< HEAD - Upstream Assets ({assetNode.dependencies.length}) - - - - Downstream Assets ({assetNode.dependedBy.length}) - - -======= Related Assets - + ->>>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 ); }; -const ASSET_NODE_DEFINITION_RUNS_QUERY = gql` - query AssetNodeDefinitionRunsQuery( - $pipelineSelector: PipelineSelector! - $repositorySelector: RepositorySelector! - ) { - repositoryOrError(repositorySelector: $repositorySelector) { - __typename - ... on Repository { - id - name - inProgressRunsByStep { - stepKey - runs { - id - runId - } - } - } - } - } -`; - export const ASSET_NODE_DEFINITION_FRAGMENT = gql` fragment AssetNodeDefinitionFragment on AssetNode { id ...AssetNodeFragment -<<<<<<< HEAD ...AssetNodeLiveFragment -======= ->>>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 dependencies { asset { id -<<<<<<< HEAD opName ...AssetNodeFragment ...AssetNodeLiveFragment -======= - ...AssetNodeFragment ->>>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 } } dependedBy { asset { id -<<<<<<< HEAD opName ...AssetNodeFragment ...AssetNodeLiveFragment -======= - ...AssetNodeFragment ->>>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 } } } ${ASSET_NODE_FRAGMENT} -<<<<<<< HEAD ${ASSET_NODE_LIVE_FRAGMENT} -======= ->>>>>>> 02814dbbf434c6974c4272f19e525dc71cb254e9 `; - -const AssetList: React.FC<{ - items: {asset: AssetNodeFragment}[]; - liveDataByNode: LiveData; -}> = ({items, liveDataByNode}) => { - const history = useHistory(); - - return ( - - {items.map((dep) => ( -
{ - if (e.isDefaultPrevented()) { - return; - } - history.push(`/instance/assets/${assetKeyToString(dep.asset.assetKey)}`); - }} - > - -
- ))} - - ); -}; diff --git a/js_modules/dagit/packages/core/src/assets/AssetView.tsx b/js_modules/dagit/packages/core/src/assets/AssetView.tsx index 1689fb8ea63d8..2933a53231c90 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetView.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetView.tsx @@ -9,11 +9,17 @@ import {Box} from '../ui/Box'; import {ButtonLink} from '../ui/ButtonLink'; import {ColorsWIP} from '../ui/Colors'; import {Spinner} from '../ui/Spinner'; -import {assetKeyToString} from '../workspace/asset-graph/Utils'; +import {useRepositoryOptions} from '../workspace/WorkspaceContext'; +import {assetKeyToString, IN_PROGRESS_RUNS_FRAGMENT} from '../workspace/asset-graph/Utils'; +import {findRepoContainingPipeline} from '../workspace/findRepoContainingPipeline'; import {AssetMaterializations} from './AssetMaterializations'; import {AssetNodeDefinition, ASSET_NODE_DEFINITION_FRAGMENT} from './AssetNodeDefinition'; import {AssetKey} from './types'; +import { + AssetNodeDefinitionRunsQuery, + AssetNodeDefinitionRunsQueryVariables, +} from './types/AssetNodeDefinitionRunsQuery'; import {AssetQuery, AssetQueryVariables} from './types/AssetQuery'; interface Props { @@ -30,6 +36,7 @@ export const AssetView: React.FC = ({assetKey}) => { const {data, loading} = useQuery(ASSET_QUERY, { variables: {assetKey: {path: assetKey.path}}, + pollInterval: 5 * 1000, }); const [params, setParams] = useQueryPersistedState({}); @@ -38,7 +45,47 @@ export const AssetView: React.FC = ({assetKey}) => { ); const definition = - data?.assetOrError && data.assetOrError.__typename === 'Asset' && data.assetOrError.definition; + data?.assetOrError && data.assetOrError.__typename === 'Asset' + ? data.assetOrError.definition + : null; + + const {options} = useRepositoryOptions(); + const [repo] = definition?.jobName ? findRepoContainingPipeline(options, definition.jobName) : []; + + const liveResult = useQuery( + ASSET_NODE_DEFINITION_RUNS_QUERY, + { + skip: !definition?.jobName, + variables: { + repositorySelector: { + repositoryLocationName: repo?.repositoryLocation.name || '', + repositoryName: repo?.repository.name || '', + }, + }, + notifyOnNetworkStatusChange: true, + pollInterval: 5 * 1000, + }, + ); + + let inProgressRuns = + liveResult.data?.repositoryOrError.__typename === 'Repository' + ? liveResult.data.repositoryOrError.inProgressRunsByStep + : []; + + inProgressRuns = [ + { + __typename: 'InProgressRunsByStep', + stepKey: 'comment_daily_stats', + runs: [ + { + __typename: 'Run', + id: 'd8998e43-b99c-4ce2-8796-17e50cbb9d2f', + runId: 'd8998e43-b99c-4ce2-8796-17e50cbb9d2f', + }, + ], + }, + ]; + console.log(inProgressRuns); return (
@@ -79,7 +126,7 @@ export const AssetView: React.FC = ({assetKey}) => { /> ) : definition ? ( - + ) : undefined}
= ({assetKey}) => { params={params} paramsTimeWindowOnly={navigatedDirectlyToTime} setParams={setParams} + inProgressRunIds={ + definition?.opName && inProgressRuns + ? inProgressRuns.find((f) => f.stepKey === definition.opName)?.runs.map((r) => r.runId) + : [] + } /> ); @@ -114,3 +166,19 @@ const ASSET_QUERY = gql` } ${ASSET_NODE_DEFINITION_FRAGMENT} `; + +const ASSET_NODE_DEFINITION_RUNS_QUERY = gql` + query AssetNodeDefinitionRunsQuery($repositorySelector: RepositorySelector!) { + repositoryOrError(repositorySelector: $repositorySelector) { + __typename + ... on Repository { + id + name + inProgressRunsByStep { + ...InProgressRunsFragment + } + } + } + } + ${IN_PROGRESS_RUNS_FRAGMENT} +`; diff --git a/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionRunsQuery.ts b/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionRunsQuery.ts index bb632c8fe2b03..4da3d75db7fd1 100644 --- a/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionRunsQuery.ts +++ b/js_modules/dagit/packages/core/src/assets/types/AssetNodeDefinitionRunsQuery.ts @@ -3,7 +3,7 @@ // @generated // This file was automatically generated and should not be edited. -import { PipelineSelector, RepositorySelector } from "./../../types/globalTypes"; +import { RepositorySelector } from "./../../types/globalTypes"; // ==================================================== // GraphQL query operation: AssetNodeDefinitionRunsQuery @@ -39,6 +39,5 @@ export interface AssetNodeDefinitionRunsQuery { } export interface AssetNodeDefinitionRunsQueryVariables { - pipelineSelector: PipelineSelector; repositorySelector: RepositorySelector; } diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index ae25873047414..2a7cabf1b6894 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -34,6 +34,7 @@ import { buildLiveData, GraphData, graphHasCycles, + IN_PROGRESS_RUNS_FRAGMENT, layoutGraph, LiveData, Node, @@ -366,11 +367,7 @@ const ASSETS_GRAPH_LIVE_QUERY = gql` id name inProgressRunsByStep { - stepKey - runs { - id - runId - } + ...InProgressRunsFragment } } } @@ -384,6 +381,7 @@ const ASSETS_GRAPH_LIVE_QUERY = gql` } } } + ${IN_PROGRESS_RUNS_FRAGMENT} ${ASSET_NODE_LIVE_FRAGMENT} `; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx index a56f428bbade7..1c9528656175f 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx @@ -16,6 +16,8 @@ import {ColorsWIP} from '../../ui/Colors'; import {IconWIP} from '../../ui/Icon'; import {markdownToPlaintext} from '../../ui/Markdown'; import {MenuItemWIP, MenuWIP} from '../../ui/Menu'; +import {Spinner} from '../../ui/Spinner'; +import {Tooltip} from '../../ui/Tooltip'; import {FontFamily} from '../../ui/styles'; import {RepoAddress} from '../types'; import {workspacePath, workspacePipelinePathGuessRepo} from '../workspacePath'; @@ -70,6 +72,23 @@ export const AssetNode: React.FC<{ {assetKeyToString(definition.assetKey)}
+ {liveData && liveData.inProgressRunIds.length > 0 && ( + + Run ID:{' '} + {liveData.inProgressRunIds.map((runId) => ( + + {runId} + + ))} +
+ } + > + + + )} + {liveData?.computeStatus === 'old' && ( upstream diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/InProgressRunsBanner.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/InProgressRunsBanner.tsx new file mode 100644 index 0000000000000..bfeeb3305022a --- /dev/null +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/InProgressRunsBanner.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import {Link} from 'react-router-dom'; + +import {BaseTag} from '../../ui/BaseTag'; +import {Box} from '../../ui/Box'; +import {ColorsWIP} from '../../ui/Colors'; +import {Spinner} from '../../ui/Spinner'; + +export const InProgressRunsBanner: React.FC<{runIds: string[]}> = ({runIds}) => { + if (runIds.length === 0) { + return null; + } + return ( + + {runIds.map((runId) => ( + + + {`Run: ${runId.slice(0, 8)}`} + + } + /> + ))} + {runIds.length === 1 ? 'is' : 'are'} currently refreshing this asset. + + ); +}; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index 8dbf8d8230ad5..1e93c55fdb180 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -8,13 +8,12 @@ import {Description} from '../../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../../pipelines/SidebarComponents'; import {GraphExplorerSolidHandleFragment_solid_definition} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; import {pluginForMetadata} from '../../plugins'; -import {BaseTag} from '../../ui/BaseTag'; import {Box} from '../../ui/Box'; import {ColorsWIP} from '../../ui/Colors'; import {IconWIP} from '../../ui/Icon'; -import {Spinner} from '../../ui/Spinner'; import {RepoAddress} from '../types'; +import {InProgressRunsBanner} from './InProgressRunsBanner'; import {assetKeyToString, LiveDataForNode} from './Utils'; import {AssetGraphQuery_pipelineOrError_Pipeline_assetNodes} from './types/AssetGraphQuery'; @@ -40,31 +39,8 @@ export const SidebarAssetInfo: React.FC<{
- {inProgressRunIds?.length > 0 && ( - - This asset may be materialized soon by + - {inProgressRunIds.map((runId) => ( - - - {`Run: ${runId.slice(0, 8)}`} - - } - /> - ))} - - )} {lastMaterialization ? ( <> diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx index 61ef72e280d03..5029b63a3d7c7 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx @@ -1,5 +1,6 @@ import {pathVerticalDiagonal} from '@vx/shape'; import * as dagre from 'dagre'; +import {gql} from 'graphql.macro'; import {AssetNodeDefinitionFragment} from '../../assets/types/AssetNodeDefinitionFragment'; @@ -11,6 +12,7 @@ import { AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetKey, } from './types/AssetGraphQuery'; import {AssetNodeLiveFragment} from './types/AssetNodeLiveFragment'; +import {InProgressRunsFragment} from './types/InProgressRunsFragment'; type AssetNode = AssetGraphQuery_pipelineOrError_Pipeline_assetNodes; type AssetKey = AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetKey; @@ -268,7 +270,7 @@ export interface LiveData { export const buildLiveData = ( graph: ReturnType, nodes: AssetNodeLiveFragment[], - inProgressRunsByStep: {stepKey: string; runs: {runId: string}[]}[], + inProgressRunsByStep: InProgressRunsFragment[], ) => { const data: LiveData = {}; @@ -314,3 +316,13 @@ function findComputeStatusForId( ? 'old' : 'good'; } + +export const IN_PROGRESS_RUNS_FRAGMENT = gql` + fragment InProgressRunsFragment on InProgressRunsByStep { + stepKey + runs { + id + runId + } + } +`; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/InProgressRunsFragment.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/InProgressRunsFragment.ts new file mode 100644 index 0000000000000..ebac19ae23072 --- /dev/null +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/InProgressRunsFragment.ts @@ -0,0 +1,20 @@ +/* tslint:disable */ +/* eslint-disable */ +// @generated +// This file was automatically generated and should not be edited. + +// ==================================================== +// GraphQL fragment: InProgressRunsFragment +// ==================================================== + +export interface InProgressRunsFragment_runs { + __typename: "Run"; + id: string; + runId: string; +} + +export interface InProgressRunsFragment { + __typename: "InProgressRunsByStep"; + stepKey: string; + runs: InProgressRunsFragment_runs[]; +} From 48c9b2af059ebb7b7987782f0fcd245133b689a6 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 16:27:32 -0600 Subject: [PATCH 16/22] Only show spinner for runs that have not yet materialized --- .../core/src/assets/AssetMaterializations.tsx | 54 ++++++- .../packages/core/src/assets/AssetView.tsx | 27 +--- .../assets/LastMaterializationMetadata.tsx | 144 +++++++++--------- .../src/workspace/asset-graph/AssetNode.tsx | 12 +- .../asset-graph/SidebarAssetInfo.tsx | 52 +------ 5 files changed, 144 insertions(+), 145 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index 514e42c425062..29a13306aaec2 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -1,22 +1,26 @@ import {gql, useQuery} from '@apollo/client'; +import styled from '@emotion/styled-base'; import flatMap from 'lodash/flatMap'; import uniq from 'lodash/uniq'; import * as React from 'react'; +import {Link} from 'react-router-dom'; +import {SidebarSection} from '../pipelines/SidebarComponents'; import {METADATA_ENTRY_FRAGMENT} from '../runs/MetadataEntry'; import {Box} from '../ui/Box'; import {ButtonGroup} from '../ui/ButtonGroup'; import {ColorsWIP} from '../ui/Colors'; +import {IconWIP} from '../ui/Icon'; import {NonIdealState} from '../ui/NonIdealState'; import {Spinner} from '../ui/Spinner'; import {Subheading} from '../ui/Text'; import {InProgressRunsBanner} from '../workspace/asset-graph/InProgressRunsBanner'; -import {InProgressRunsFragment} from '../workspace/asset-graph/types/InProgressRunsFragment'; import {ASSET_LINEAGE_FRAGMENT} from './AssetLineageElements'; import {AssetMaterializationTable} from './AssetMaterializationTable'; import {AssetValueGraph} from './AssetValueGraph'; import {AssetViewParams} from './AssetView'; +import {LatestMaterializationMetadata} from './LastMaterializationMetadata'; import {AssetKey, AssetNumericHistoricalData} from './types'; import {AssetMaterializationFragment} from './types/AssetMaterializationFragment'; import { @@ -39,8 +43,8 @@ export const AssetMaterializations: React.FC = ({ assetKey, asSidebarSection, params, - setParams, paramsTimeWindowOnly, + setParams, inProgressRunIds, }) => { const {data, loading} = useQuery( @@ -93,9 +97,43 @@ export const AssetMaterializations: React.FC = ({ ); } + const notYetMaterializedRunIds = (inProgressRunIds || []).filter( + (runId) => + !materializations.some( + (m) => m.runOrError.__typename === 'Run' && m.runOrError.runId === runId, + ), + ); + if (asSidebarSection) { + const latest = materializations[0]; return ( - + <> + + + {latest ? ( + <> +
+ +
+ + + {'View All in Asset Catalog '} + + + + + ) : ( + — + )} + + + + + ); } @@ -121,7 +159,7 @@ export const AssetMaterializations: React.FC = ({
) : null} - {inProgressRunIds && } + = ({assetKey}) => { useDocumentTitle(`Asset: ${assetKeyToString(assetKey)}`); - const {data, loading} = useQuery(ASSET_QUERY, { - variables: {assetKey: {path: assetKey.path}}, - pollInterval: 5 * 1000, - }); - const [params, setParams] = useQueryPersistedState({}); const [navigatedDirectlyToTime, setNavigatedDirectlyToTime] = React.useState(() => Boolean(params.asOf), ); + const {data, loading} = useQuery(ASSET_QUERY, { + variables: {assetKey: {path: assetKey.path}}, + pollInterval: 5 * 1000, + }); + const definition = data?.assetOrError && data.assetOrError.__typename === 'Asset' ? data.assetOrError.definition @@ -67,26 +67,11 @@ export const AssetView: React.FC = ({assetKey}) => { }, ); - let inProgressRuns = + const inProgressRuns = liveResult.data?.repositoryOrError.__typename === 'Repository' ? liveResult.data.repositoryOrError.inProgressRunsByStep : []; - inProgressRuns = [ - { - __typename: 'InProgressRunsByStep', - stepKey: 'comment_daily_stats', - runs: [ - { - __typename: 'Run', - id: 'd8998e43-b99c-4ce2-8796-17e50cbb9d2f', - runId: 'd8998e43-b99c-4ce2-8796-17e50cbb9d2f', - }, - ], - }, - ]; - console.log(inProgressRuns); - return (
diff --git a/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx b/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx index 1bef14dc1abd7..dabecfa1966f1 100644 --- a/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx +++ b/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx @@ -69,86 +69,84 @@ export const LatestMaterializationMetadata: React.FC<{ const latestAssetLineage = latestEvent?.assetLineage; return ( - - -
+ + + + + + + {latest?.partition ? ( - + + + + ) : null} + + + + + {latestAssetLineage?.length ? ( + + - {latest?.partition ? ( - - - - - ) : null} - - + ) : null} + {latestEvent?.materialization.metadataEntries.map((entry) => ( + + - {latestAssetLineage?.length ? ( - - - - - ) : null} - {latestEvent?.materialization.metadataEntries.map((entry) => ( - - - - - ))} - - - + ))} + + ); }; diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx index 1c9528656175f..f4848f081898c 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx @@ -36,8 +36,12 @@ export const AssetNode: React.FC<{ }> = React.memo(({definition, metadata, selected, liveData, repoAddress, secondaryHighlight}) => { const launch = useLaunchSingleAssetJob(); const history = useHistory(); + const {materializationEvent: event, runOrError} = liveData?.lastMaterialization || {}; const kind = metadata.find((m) => m.key === 'kind')?.value; + const notYetMaterializedRunIds = (liveData?.inProgressRunIds || []).filter( + (runId) => (runOrError?.__typename === 'Run' ? runOrError.runId : '') !== runId, + ); return ( - {assetKeyToString(definition.assetKey)} +
+ {assetKeyToString(definition.assetKey)} +
- {liveData && liveData.inProgressRunIds.length > 0 && ( + {notYetMaterializedRunIds.length > 0 && ( Run ID:{' '} - {liveData.inProgressRunIds.map((runId) => ( + {notYetMaterializedRunIds.map((runId) => ( {runId} diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index 1e93c55fdb180..05ef6ad1e13f8 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -1,19 +1,14 @@ import React from 'react'; -import {Link} from 'react-router-dom'; -import styled from 'styled-components/macro'; import {AssetMaterializations} from '../../assets/AssetMaterializations'; -import {LatestMaterializationMetadata} from '../../assets/LastMaterializationMetadata'; import {Description} from '../../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../../pipelines/SidebarComponents'; import {GraphExplorerSolidHandleFragment_solid_definition} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; import {pluginForMetadata} from '../../plugins'; import {Box} from '../../ui/Box'; import {ColorsWIP} from '../../ui/Colors'; -import {IconWIP} from '../../ui/Icon'; import {RepoAddress} from '../types'; -import {InProgressRunsBanner} from './InProgressRunsBanner'; import {assetKeyToString, LiveDataForNode} from './Utils'; import {AssetGraphQuery_pipelineOrError_Pipeline_assetNodes} from './types/AssetGraphQuery'; @@ -24,7 +19,6 @@ export const SidebarAssetInfo: React.FC<{ repoAddress: RepoAddress; }> = ({node, definition, repoAddress, liveData}) => { const Plugin = pluginForMetadata(definition.metadata); - const {lastMaterialization, inProgressRunIds} = liveData || {}; return (
@@ -39,45 +33,15 @@ export const SidebarAssetInfo: React.FC<{
- - - {lastMaterialization ? ( - <> -
- -
- - - {'View All in Asset Catalog '} - - - - - ) : ( - — - )} - - - {lastMaterialization ? ( - - {}} - /> - - ) : null} + {}} + />
); }; - -const AssetCatalogLink = styled(Link)` - display: flex; - gap: 5px; - align-items: center; - justify-content: flex-end; - margin-top: -10px; -`; From 0db26159e36770e5259a76242d7be7d7f38c1e76 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 17:45:33 -0600 Subject: [PATCH 17/22] Foreign node CSV should never be considered older than data --- .../packages/core/src/workspace/asset-graph/Utils.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx index 5029b63a3d7c7..2c9c636a531bb 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx @@ -186,6 +186,7 @@ export const layoutGraph = ( if ( graphData.nodes[downstreamId] && graphData.nodes[downstreamId].hidden && + graphData.nodes[upstreamId] && graphData.nodes[upstreamId].hidden ) { return; @@ -260,7 +261,7 @@ export type Status = 'good' | 'old' | 'none' | 'unknown'; export interface LiveDataForNode { computeStatus: Status; inProgressRunIds: string[]; - lastMaterialization: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations; + lastMaterialization: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations | null; lastStepStart: number; } export interface LiveData { @@ -275,15 +276,16 @@ export const buildLiveData = ( const data: LiveData = {}; for (const node of nodes) { - const lastMaterialization = node.assetMaterializations[0]; + const lastMaterialization = node.assetMaterializations[0] || null; const lastStepStart = lastMaterialization?.materializationEvent.stepStats?.startTime || 0; + const isForeignNode = !node.opName; data[node.id] = { lastStepStart, lastMaterialization, inProgressRunIds: inProgressRunsByStep.find((r) => r.stepKey === node.opName)?.runs.map((r) => r.runId) || [], - computeStatus: lastMaterialization ? 'unknown' : 'none', + computeStatus: isForeignNode ? 'good' : lastMaterialization ? 'unknown' : 'none', }; } From c21c23cfe10c8d332f28ceb4c9e292d633497f50 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 18:29:01 -0600 Subject: [PATCH 18/22] Make assets with no opName unclickable --- .../core/src/assets/AssetNeighborsGraph.tsx | 13 ++++++------- .../core/src/workspace/asset-graph/Utils.tsx | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx index 6f170f755471a..6e0c6a46c0b32 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNeighborsGraph.tsx @@ -79,18 +79,17 @@ export const AssetNeighborsGraph: React.FC<{ const graphNode = graphData.nodes[layoutNode.id]; return ( { e.stopPropagation(); - history.push(`/instance/assets/${assetKeyToString(graphNode.assetKey)}`); + if (graphNode.definition.opName) { + history.push(`/instance/assets/${assetKeyToString(graphNode.assetKey)}`); + } }} - style={{overflow: 'visible'}} > - {graphNode.hidden ? ( + {graphNode.hidden || !graphNode.definition.opName ? ( ) : ( ({})); + const shouldRender = (node?: Node) => node && !node.hidden && node.definition.opName; + Object.values(graphData.nodes) - .filter((x) => !x.hidden) + .filter(shouldRender) .forEach((node) => { const {width, height} = getNodeDimensions(node.definition); g.setNode(node.id, {width: opts.mini ? 230 : width, height}); @@ -184,24 +186,23 @@ export const layoutGraph = ( const downstreamIds = Object.keys(graphData.downstream[upstreamId]); downstreamIds.forEach((downstreamId) => { if ( - graphData.nodes[downstreamId] && - graphData.nodes[downstreamId].hidden && - graphData.nodes[upstreamId] && - graphData.nodes[upstreamId].hidden + !shouldRender(graphData.nodes[downstreamId]) && + !shouldRender(graphData.nodes[upstreamId]) ) { return; } g.setEdge({v: upstreamId, w: downstreamId}, {weight: 1}); - if (!graphData.nodes[downstreamId] || graphData.nodes[downstreamId].hidden) { + + if (!shouldRender(graphData.nodes[downstreamId])) { foreignNodes[downstreamId] = true; - } else if (!graphData.nodes[upstreamId] || graphData.nodes[upstreamId].hidden) { + } else if (!shouldRender(graphData.nodes[upstreamId])) { foreignNodes[upstreamId] = true; } }); }); - Object.keys(foreignNodes).forEach((upstreamId) => { - g.setNode(upstreamId, getForeignNodeDimensions(upstreamId)); + Object.keys(foreignNodes).forEach((id) => { + g.setNode(id, getForeignNodeDimensions(id)); }); dagre.layout(g); From b887620476b8a39af8dff866413904e240c3479e Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 18:33:45 -0600 Subject: [PATCH 19/22] Autorefresh the entire asset details page, fix flickering --- .../core/src/assets/AssetEntryRoot.tsx | 82 ++++--------------- .../core/src/assets/AssetMaterializations.tsx | 34 +++++--- .../core/src/assets/AssetNodeDefinition.tsx | 22 +++-- .../core/src/assets/AssetPageHeader.tsx | 69 ++++++++++++++++ .../packages/core/src/assets/AssetView.tsx | 44 +++++++--- .../core/src/assets/types/AssetQuery.ts | 11 +++ .../dagit/packages/core/src/runs/RunUtils.tsx | 10 +++ .../asset-graph/AssetGraphExplorer.tsx | 20 ++--- .../asset-graph/SidebarAssetInfo.tsx | 4 +- 9 files changed, 187 insertions(+), 109 deletions(-) create mode 100644 js_modules/dagit/packages/core/src/assets/AssetPageHeader.tsx diff --git a/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx b/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx index 9b4052f590937..f59c519192eb3 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx @@ -1,21 +1,14 @@ import {gql, useQuery} from '@apollo/client'; -import {BreadcrumbProps, Breadcrumbs} from '@blueprintjs/core'; import * as React from 'react'; -import {Link, RouteComponentProps} from 'react-router-dom'; -import styled from 'styled-components/macro'; +import {RouteComponentProps} from 'react-router-dom'; -import {Box} from '../ui/Box'; -import {ColorsWIP} from '../ui/Colors'; import {Loading} from '../ui/Loading'; import {Page} from '../ui/Page'; -import {PageHeader} from '../ui/PageHeader'; -import {TagWIP} from '../ui/TagWIP'; -import {Heading} from '../ui/Text'; +import {AssetPageHeader} from './AssetPageHeader'; import {AssetView} from './AssetView'; import {AssetsCatalogTable} from './AssetsCatalogTable'; import {AssetEntryRootQuery} from './types/AssetEntryRootQuery'; -import {useAssetView} from './useAssetView'; export const AssetEntryRoot: React.FC = ({match}) => { const currentPath: string[] = (match.params['0'] || '') @@ -23,74 +16,27 @@ export const AssetEntryRoot: React.FC = ({match}) => { .filter((x: string) => x) .map(decodeURIComponent); - const [view] = useAssetView(); - const queryResult = useQuery(ASSET_ENTRY_ROOT_QUERY, { variables: {assetKey: {path: currentPath}}, }); - const breadcrumbs = React.useMemo(() => { - if (currentPath.length === 1 || view !== 'directory') { - return null; - } - - const list: BreadcrumbProps[] = []; - currentPath.reduce((accum: string, elem: string) => { - const href = `${accum}/${encodeURIComponent(elem)}`; - list.push({text: elem, href}); - return href; - }, '/instance/assets'); - - return list; - }, [currentPath, view]); - - return ( + return queryResult.loading ? ( - {currentPath[currentPath.length - 1]} - ) : ( - - ( - - {text} - - )} - currentBreadcrumbRenderer={({text}) => {text}} - /> - - ) - } - tags={Asset} - /> - - {({assetOrError}) => { - if (assetOrError.__typename === 'AssetNotFoundError') { - return ; - } - - return ; - }} - + + {() => null} + + ) : queryResult.data?.assetOrError.__typename === 'AssetNotFoundError' ? ( + + + + + ) : ( + + ); }; -const BreadcrumbLink = styled(Link)` - color: ${ColorsWIP.Gray800}; - - :hover, - :active { - color: ${ColorsWIP.Gray800}; - } -`; - const ASSET_ENTRY_ROOT_QUERY = gql` query AssetEntryRootQuery($assetKey: AssetKeyInput!) { assetOrError(assetKey: $assetKey) { diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index 29a13306aaec2..c45ff0363326b 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -1,9 +1,9 @@ import {gql, useQuery} from '@apollo/client'; -import styled from '@emotion/styled-base'; import flatMap from 'lodash/flatMap'; import uniq from 'lodash/uniq'; import * as React from 'react'; import {Link} from 'react-router-dom'; +import styled from 'styled-components/macro'; import {SidebarSection} from '../pipelines/SidebarComponents'; import {METADATA_ENTRY_FRAGMENT} from '../runs/MetadataEntry'; @@ -36,27 +36,41 @@ interface Props { params: AssetViewParams; paramsTimeWindowOnly: boolean; setParams: (params: AssetViewParams) => void; + + // This timestamp is a "hint", when it changes this component will refetch + // to retrieve new data. Just don't want to poll the entire table query. + assetLastMaterializedAt: string | undefined; } + const LABEL_STEP_EXECUTION_TIME = 'Step Execution Time'; export const AssetMaterializations: React.FC = ({ assetKey, + assetLastMaterializedAt, asSidebarSection, params, paramsTimeWindowOnly, setParams, inProgressRunIds, }) => { - const {data, loading} = useQuery( - ASSET_MATERIALIZATIONS_QUERY, - { - variables: { - assetKey: {path: assetKey.path}, - before: paramsTimeWindowOnly && params.asOf ? `${Number(params.asOf) + 1}` : undefined, - limit: 200, - }, + const {data, loading, refetch} = useQuery< + AssetMaterializationsQuery, + AssetMaterializationsQueryVariables + >(ASSET_MATERIALIZATIONS_QUERY, { + variables: { + assetKey: {path: assetKey.path}, + before: paramsTimeWindowOnly && params.asOf ? `${Number(params.asOf) + 1}` : undefined, + limit: 200, }, - ); + }); + + React.useEffect(() => { + if (paramsTimeWindowOnly) { + return; + } + console.log('refetch'); + refetch(); + }, [paramsTimeWindowOnly, assetLastMaterializedAt, refetch]); const asset = data?.assetOrError.__typename === 'Asset' ? data?.assetOrError : null; const materializations = asset?.assetMaterializations || []; diff --git a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx index a939c1ee3f79b..8597fe53f166c 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx @@ -5,23 +5,32 @@ import {Description} from '../pipelines/Description'; import {PipelineReference} from '../pipelines/PipelineReference'; import {Box} from '../ui/Box'; import {ColorsWIP} from '../ui/Colors'; +import {NonIdealState} from '../ui/NonIdealState'; import {Subheading} from '../ui/Text'; -import {useRepositoryOptions} from '../workspace/WorkspaceContext'; +import {DagsterRepoOption} from '../workspace/WorkspaceContext'; import {ASSET_NODE_FRAGMENT, ASSET_NODE_LIVE_FRAGMENT} from '../workspace/asset-graph/AssetNode'; import {buildGraphDataFromSingleNode, buildLiveData} from '../workspace/asset-graph/Utils'; import {InProgressRunsFragment} from '../workspace/asset-graph/types/InProgressRunsFragment'; -import {findRepoContainingPipeline} from '../workspace/findRepoContainingPipeline'; import {AssetNeighborsGraph} from './AssetNeighborsGraph'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinitionFragment'; export const AssetNodeDefinition: React.FC<{ + repo: DagsterRepoOption | null; assetNode: AssetNodeDefinitionFragment; inProgressRuns: InProgressRunsFragment[]; -}> = ({assetNode, inProgressRuns}) => { - const {options} = useRepositoryOptions(); - const [repo] = assetNode?.jobName ? findRepoContainingPipeline(options, assetNode.jobName) : []; - +}> = ({repo, assetNode, inProgressRuns}) => { + if (!repo) { + return ( + + + + ); + } const nodesWithLatestMaterialization = [ assetNode, ...assetNode.dependencies.map((d) => d.asset), @@ -32,7 +41,6 @@ export const AssetNodeDefinition: React.FC<{ nodesWithLatestMaterialization, inProgressRuns, ); - console.log(liveDataByNode); return ( > +> = ({currentPath, ...extra}) => { + const [view] = useAssetView(); + + const breadcrumbs = React.useMemo(() => { + if (currentPath.length === 1 || view !== 'directory') { + return null; + } + + const list: BreadcrumbProps[] = []; + currentPath.reduce((accum: string, elem: string) => { + const href = `${accum}/${encodeURIComponent(elem)}`; + list.push({text: elem, href}); + return href; + }, '/instance/assets'); + + return list; + }, [currentPath, view]); + + return ( + {currentPath[currentPath.length - 1]} + ) : ( + + ( + + {text} + + )} + currentBreadcrumbRenderer={({text}) => {text}} + /> + + ) + } + tags={Asset} + {...extra} + /> + ); +}; + +const BreadcrumbLink = styled(Link)` + color: ${ColorsWIP.Gray800}; + + :hover, + :active { + color: ${ColorsWIP.Gray800}; + } +`; diff --git a/js_modules/dagit/packages/core/src/assets/AssetView.tsx b/js_modules/dagit/packages/core/src/assets/AssetView.tsx index 74f8d4c006a4a..336589117f329 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetView.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetView.tsx @@ -1,9 +1,11 @@ import {gql, useQuery} from '@apollo/client'; import * as React from 'react'; +import {QueryCountdown} from '../app/QueryCountdown'; import {Timestamp} from '../app/time/Timestamp'; import {useDocumentTitle} from '../hooks/useDocumentTitle'; import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; +import {useDidLaunchEvent} from '../runs/RunUtils'; import {Alert} from '../ui/Alert'; import {Box} from '../ui/Box'; import {ButtonLink} from '../ui/ButtonLink'; @@ -15,6 +17,7 @@ import {findRepoContainingPipeline} from '../workspace/findRepoContainingPipelin import {AssetMaterializations} from './AssetMaterializations'; import {AssetNodeDefinition, ASSET_NODE_DEFINITION_FRAGMENT} from './AssetNodeDefinition'; +import {AssetPageHeader} from './AssetPageHeader'; import {AssetKey} from './types'; import { AssetNodeDefinitionRunsQuery, @@ -39,20 +42,25 @@ export const AssetView: React.FC = ({assetKey}) => { Boolean(params.asOf), ); - const {data, loading} = useQuery(ASSET_QUERY, { + const queryResult = useQuery(ASSET_QUERY, { variables: {assetKey: {path: assetKey.path}}, + notifyOnNetworkStatusChange: true, pollInterval: 5 * 1000, }); - const definition = - data?.assetOrError && data.assetOrError.__typename === 'Asset' - ? data.assetOrError.definition - : null; + // Refresh immediately when a run is launched from this page + useDidLaunchEvent(queryResult.refetch); + const {assetOrError} = queryResult.data || queryResult.previousData || {}; + const asset = assetOrError && assetOrError.__typename === 'Asset' ? assetOrError : null; + const definition = asset?.definition; + const lastMaterializedAt = asset?.assetMaterializations[0]?.materializationEvent.timestamp; + + // Note: Todo create a better way to link an Asset to a Repo const {options} = useRepositoryOptions(); const [repo] = definition?.jobName ? findRepoContainingPipeline(options, definition.jobName) : []; - const liveResult = useQuery( + const bonusResult = useQuery( ASSET_NODE_DEFINITION_RUNS_QUERY, { skip: !definition?.jobName, @@ -68,14 +76,23 @@ export const AssetView: React.FC = ({assetKey}) => { ); const inProgressRuns = - liveResult.data?.repositoryOrError.__typename === 'Repository' - ? liveResult.data.repositoryOrError.inProgressRunsByStep + bonusResult.data?.repositoryOrError.__typename === 'Repository' + ? bonusResult.data.repositoryOrError.inProgressRunsByStep : []; return (
+ + +
+ } + /> +
- {loading ? ( + {queryResult.loading && !queryResult.previousData ? ( = ({assetKey}) => { /> ) : definition ? ( - + ) : undefined}
void; }>({refetch: () => {}}); +export function useDidLaunchEvent(cb: () => void) { + React.useEffect(() => { + document.addEventListener('run-launched', cb); + return () => { + document.removeEventListener('run-launched', cb); + }; + }, [cb]); +} + export function handleLaunchResult( basePath: string, pipelineName: string, @@ -52,6 +61,7 @@ export function handleLaunchResult( } else { window.location.href = url; } + document.dispatchEvent(new CustomEvent('run-launched')); } else if (obj.__typename === 'PythonError') { showCustomAlert({ title: 'Error', diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index 2a7cabf1b6894..671f73dca14f9 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -12,6 +12,7 @@ import {useDocumentTitle} from '../../hooks/useDocumentTitle'; import {ExplorerPath} from '../../pipelines/PipelinePathUtils'; import {SidebarPipelineOrJobOverview} from '../../pipelines/SidebarPipelineOrJobOverview'; import {GraphExplorerSolidHandleFragment} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; +import {useDidLaunchEvent} from '../../runs/RunUtils'; import {GraphQueryInput} from '../../ui/GraphQueryInput'; import {Loading} from '../../ui/Loading'; import {NonIdealState} from '../../ui/NonIdealState'; @@ -80,6 +81,7 @@ export const AssetGraphExplorer: React.FC = (props) => { ); useDocumentTitle('Assets'); + useDidLaunchEvent(liveResult.refetch); const graphData = React.useMemo(() => { if (queryResult.data?.pipelineOrError.__typename !== 'Pipeline') { @@ -131,6 +133,7 @@ export const AssetGraphExplorer: React.FC = (props) => { return ( { const graphNode = graphData.nodes[layoutNode.id]; - const {width, height} = - !graphNode || graphNode.hidden - ? getForeignNodeDimensions(layoutNode.id) - : getNodeDimensions(graphNode.definition); - const path = JSON.parse(layoutNode.id); if ( - layoutNode.x + width < bounds.left || - layoutNode.y + height < bounds.top || + layoutNode.x + layoutNode.width < bounds.left || + layoutNode.y + layoutNode.height < bounds.top || layoutNode.x > bounds.right || layoutNode.y > bounds.bottom ) { @@ -275,11 +272,8 @@ const AssetGraphExplorerWithData: React.FC< return ( onSelectNode(e, {path}, graphNode)} style={{overflow: 'visible'}} > diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index 05ef6ad1e13f8..72f1b63754c2c 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -19,6 +19,7 @@ export const SidebarAssetInfo: React.FC<{ repoAddress: RepoAddress; }> = ({node, definition, repoAddress, liveData}) => { const Plugin = pluginForMetadata(definition.metadata); + const {inProgressRunIds, lastMaterialization} = liveData || {}; return (
@@ -36,8 +37,9 @@ export const SidebarAssetInfo: React.FC<{ {}} From a09624962ac095bd02ce0ffab9d5933b81436922 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 18:36:23 -0600 Subject: [PATCH 20/22] Fix linter issues --- .../src/workspace/asset-graph/AssetGraphExplorer.tsx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index 671f73dca14f9..d9c022815da41 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -22,13 +22,8 @@ import {repoAddressToSelector} from '../repoAddressToSelector'; import {RepoAddress} from '../types'; import {AssetLinks} from './AssetLinks'; -import { - AssetNode, - ASSET_NODE_FRAGMENT, - ASSET_NODE_LIVE_FRAGMENT, - getNodeDimensions, -} from './AssetNode'; -import {ForeignNode, getForeignNodeDimensions} from './ForeignNode'; +import {AssetNode, ASSET_NODE_FRAGMENT, ASSET_NODE_LIVE_FRAGMENT} from './AssetNode'; +import {ForeignNode} from './ForeignNode'; import {SidebarAssetInfo} from './SidebarAssetInfo'; import { buildGraphData, From 332a0479fbb5e98ac660c2a0d1be1bc136eebc9b Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Wed, 1 Dec 2021 17:04:20 -0600 Subject: [PATCH 21/22] Polish appearance of graphs in the left sidebar of the asset job view --- .../core/src/assets/AssetMaterializations.tsx | 90 ++++++++++++------- .../core/src/assets/AssetValueGraph.tsx | 22 +---- .../assets/LastMaterializationMetadata.tsx | 26 +----- 3 files changed, 60 insertions(+), 78 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index c45ff0363326b..bb6f4e92557d1 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -13,7 +13,7 @@ import {ColorsWIP} from '../ui/Colors'; import {IconWIP} from '../ui/Icon'; import {NonIdealState} from '../ui/NonIdealState'; import {Spinner} from '../ui/Spinner'; -import {Subheading} from '../ui/Text'; +import {Caption, Subheading} from '../ui/Text'; import {InProgressRunsBanner} from '../workspace/asset-graph/InProgressRunsBanner'; import {ASSET_LINEAGE_FRAGMENT} from './AssetLineageElements'; @@ -99,18 +99,6 @@ export const AssetMaterializations: React.FC = ({ ); } - if (!reversed.length) { - return ( - - - - ); - } - const notYetMaterializedRunIds = (inProgressRunIds || []).filter( (runId) => !materializations.some( @@ -124,21 +112,26 @@ export const AssetMaterializations: React.FC = ({ <> - {latest ? ( - <> + <> + {latest ? (
- +
- - - {'View All in Asset Catalog '} - - + ) : ( + + No materializations found - - ) : ( - — - )} + )} + + + {'View All in Asset Catalog '} + + + + = ({ ); } + if (!reversed.length) { + return ( + + + + ); + } + return ( Materializations {hasPartitions ? ( @@ -223,14 +228,33 @@ const AssetMaterializationGraphs: React.FC<{ }} > {[...graphedLabels].sort().map((label) => ( - x !== xHover && setXHover(x)} - /> + style={{width: '100%'}} + border={{side: 'bottom', width: 1, color: ColorsWIP.KeylineGray}} + > + {props.asSidebarSection ? ( + +
+ + ) : ( + + {label} + + )} + + x !== xHover && setXHover(x)} + /> + + ))} {xAxis === 'partition' && ( diff --git a/js_modules/dagit/packages/core/src/assets/AssetValueGraph.tsx b/js_modules/dagit/packages/core/src/assets/AssetValueGraph.tsx index 676c5364a0866..aab6b1e0d8e7e 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetValueGraph.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetValueGraph.tsx @@ -2,11 +2,8 @@ import {ActiveElement} from 'chart.js'; import 'chartjs-adapter-date-fns'; import * as React from 'react'; import {Line} from 'react-chartjs-2'; -import styled from 'styled-components/macro'; -import {Box} from '../ui/Box'; import {ColorsWIP} from '../ui/Colors'; -import {Subheading} from '../ui/Text'; import {AssetNumericHistoricalData} from './types'; @@ -101,22 +98,5 @@ export const AssetValueGraph: React.FC<{ }, }; - return ( - - - {props.label} - - - - - - ); + return ; }; - -const Container = styled.div` - padding: 16px 24px; - box-shadow: ${ColorsWIP.KeylineGray} 0 -1px 0 inset, ${ColorsWIP.KeylineGray} -1px 0 0 inset; -`; diff --git a/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx b/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx index dabecfa1966f1..dca70b6a19757 100644 --- a/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx +++ b/js_modules/dagit/packages/core/src/assets/LastMaterializationMetadata.tsx @@ -23,8 +23,7 @@ import {LatestMaterializationMetadataFragment} from './types/LatestMaterializati export const LatestMaterializationMetadata: React.FC<{ latest: LatestMaterializationMetadataFragment | undefined; - asOf: string | null; -}> = ({latest, asOf}) => { +}> = ({latest}) => { const latestRun = latest?.runOrError.__typename === 'Run' ? latest?.runOrError : null; const repositoryOrigin = latestRun?.repositoryOrigin; const repoAddress = repositoryOrigin @@ -33,33 +32,12 @@ export const LatestMaterializationMetadata: React.FC<{ const repo = useRepository(repoAddress); if (!latest) { - if (!asOf) { - return ( - - - - ); - } - return ( - No materializations found at{' '} - - . - - } + description="No materializations were found for this asset." /> ); From f9404552fa174d4e5fac05c3ba39c88e320b703d Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 2 Dec 2021 13:36:08 -0600 Subject: [PATCH 22/22] Rm console log --- .../dagit/packages/core/src/assets/AssetMaterializations.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index bb6f4e92557d1..62633b5193580 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -68,7 +68,6 @@ export const AssetMaterializations: React.FC = ({ if (paramsTimeWindowOnly) { return; } - console.log('refetch'); refetch(); }, [paramsTimeWindowOnly, assetLastMaterializedAt, refetch]);
PartitionMaterialization MetadataParent MaterializationsPartitionTimestamp Job / Pipeline Run
- {latest.partition || None} + <> +
+ {latest.partition || None} + + + + {predecessors?.length ? ( + + ) : null} + + + + + + + + + {stepKey} + + + + + + + + {titleForRun(run)} + + +
+ {materialization.description && ( + {materialization.description} + )} + {metadataEntries.length || hasLineage ? ( + + {(metadataEntries || []).map((entry) => ( +
{entry.label} + +
Parent Materializations + +
- {materialization.description ? ( -
{materialization.description}
- ) : null} - {metadataEntries && metadataEntries.length ? ( - - - - ) : null} - 		{} - - - {predecessors?.length ? ( - - ) : null} - - - - - - - - {titleForRun(run)} - - -
- {latest.partition || None} + + + + {latest.partition || None} + - - - {predecessors?.length ? ( - - ) : null} + + {!hasPartitions && } + + + {predecessors?.length ? ( + + ) : null} + @@ -135,7 +141,7 @@ const AssetMaterializationRow: React.FC<{
@@ -171,6 +177,11 @@ const AssetMaterializationRow: React.FC<{ ); }; +const ClickableRow = styled.tr` + &:hover { + background: ${ColorsWIP.Gray10}; + } +`; const DetailsTable = styled.table` margin: -2px -2px -3px; tr td { @@ -216,7 +227,10 @@ export const AssetPredecessorLink: React.FC = ({ ({latest: p}))} + materializations={predecessors.map((p) => ({ + latest: p, + timestamp: p.materializationEvent.timestamp, + }))} /> @@ -228,3 +242,15 @@ export const AssetPredecessorLink: React.FC = ({ ); }; + +const DisclosureTriangle: React.FC<{$open: boolean}> = ({$open}) => ( + +); diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx index 2bcef1033d962..3198c47ee7057 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx @@ -3,7 +3,6 @@ import flatMap from 'lodash/flatMap'; import uniq from 'lodash/uniq'; import * as React from 'react'; -import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; import {METADATA_ENTRY_FRAGMENT} from '../runs/MetadataEntry'; import {Box} from '../ui/Box'; import {ButtonGroup} from '../ui/ButtonGroup'; @@ -15,6 +14,7 @@ import {Subheading} from '../ui/Text'; import {ASSET_LINEAGE_FRAGMENT} from './AssetLineageElements'; import {AssetMaterializationTable} from './AssetMaterializationTable'; import {AssetValueGraph} from './AssetValueGraph'; +import {AssetViewParams} from './AssetView'; import {AssetKey, AssetNumericHistoricalData} from './types'; import {AssetMaterializationFragment} from './types/AssetMaterializationFragment'; import { @@ -25,40 +25,41 @@ import {HistoricalMaterialization, useMaterializationBuckets} from './useMateria interface Props { assetKey: AssetKey; - asOf?: string | null; asSidebarSection?: boolean; + params: AssetViewParams; + paramsTimeWindowOnly: boolean; + setParams: (params: AssetViewParams) => void; } const LABEL_STEP_EXECUTION_TIME = 'Step Execution Time'; -export const AssetMaterializations: React.FC = ({assetKey, asSidebarSection}) => { +export const AssetMaterializations: React.FC = ({ + assetKey, + asSidebarSection, + params, + setParams, + paramsTimeWindowOnly, +}) => { const {data, loading} = useQuery( ASSET_MATERIALIZATIONS_QUERY, { variables: { assetKey: {path: assetKey.path}, + before: paramsTimeWindowOnly && params.asOf ? `${Number(params.asOf) + 1}` : undefined, limit: 200, }, }, ); const asset = data?.assetOrError.__typename === 'Asset' ? data?.assetOrError : null; - const assetMaterializations = asset?.assetMaterializations || []; - const hasPartitions = assetMaterializations.some((m) => m.partition); - const hasLineage = assetMaterializations.some( - (m) => m.materializationEvent.assetLineage.length > 0, - ); - - const [{xAxis, asOf}, setParams] = useQueryPersistedState<{ - xAxis: 'partition' | 'time'; - asOf: string; - }>({ - defaults: {xAxis: hasPartitions ? 'partition' : 'time'}, - }); + const materializations = asset?.assetMaterializations || []; + const hasPartitions = materializations.some((m) => m.partition); + const hasLineage = materializations.some((m) => m.materializationEvent.assetLineage.length > 0); + const {xAxis = hasPartitions ? 'partition' : 'time', asOf} = params; const bucketed = useMaterializationBuckets({ - materializations: assetMaterializations, - hasPartitions, shouldBucketPartitions: xAxis === 'partition', + materializations, + hasPartitions, }); const reversed = React.useMemo(() => [...bucketed].reverse(), [bucketed]); @@ -111,7 +112,7 @@ export const AssetMaterializations: React.FC = ({assetKey, asSidebarSecti {id: 'partition', label: 'By partition'}, {id: 'time', label: 'By timestamp'}, ]} - onClick={(id: string) => setParams({xAxis: id as 'partition' | 'time', asOf: asOf})} + onClick={(id: string) => setParams({...params, xAxis: id as 'partition' | 'time'})} /> ) : null} @@ -121,10 +122,14 @@ export const AssetMaterializations: React.FC = ({assetKey, asSidebarSecti hasLineage={hasLineage} materializations={bucketed} focused={ - bucketed.find((b) => Number(b.latest.materializationEvent.timestamp) <= Number(asOf)) - ?.latest.materializationEvent.timestamp + (bucketed.find((b) => Number(b.timestamp) <= Number(asOf)) || bucketed[0])?.timestamp + } + setFocused={(asOf) => + setParams({ + ...params, + asOf: paramsTimeWindowOnly || asOf !== bucketed[0]?.timestamp ? asOf : undefined, + }) } - setFocused={(asOf) => setParams({xAxis, asOf: asOf})} /> @@ -148,9 +153,7 @@ const AssetMaterializationGraphs: React.FC<{ const latest = assetMaterializations.map((m) => m.latest); const graphDataByMetadataLabel = extractNumericData(latest, xAxis); - const [graphedLabels, setGraphedLabels] = React.useState(() => - Object.keys(graphDataByMetadataLabel).slice(0, 4), - ); + const [graphedLabels] = React.useState(() => Object.keys(graphDataByMetadataLabel).slice(0, 4)); return ( <> diff --git a/js_modules/dagit/packages/core/src/assets/AssetView.tsx b/js_modules/dagit/packages/core/src/assets/AssetView.tsx index e16c8354ae777..ccc45d7e7297a 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetView.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetView.tsx @@ -1,8 +1,13 @@ import {gql, useQuery} from '@apollo/client'; import * as React from 'react'; +import {Timestamp} from '../app/time/Timestamp'; import {useDocumentTitle} from '../hooks/useDocumentTitle'; +import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; +import {Alert} from '../ui/Alert'; import {Box} from '../ui/Box'; +import {ButtonLink} from '../ui/ButtonLink'; +import {ColorsWIP} from '../ui/Colors'; import {Spinner} from '../ui/Spinner'; import {assetKeyToString} from '../workspace/asset-graph/Utils'; @@ -15,6 +20,11 @@ interface Props { assetKey: AssetKey; } +export interface AssetViewParams { + xAxis?: 'partition' | 'time'; + asOf?: string; +} + export const AssetView: React.FC = ({assetKey}) => { useDocumentTitle(`Asset: ${assetKeyToString(assetKey)}`); @@ -24,28 +34,62 @@ export const AssetView: React.FC = ({assetKey}) => { }, }); + const [params, setParams] = useQueryPersistedState({}); + const [navigatedDirectlyToTime, setNavigatedDirectlyToTime] = React.useState(() => + Boolean(params.asOf), + ); + + const definition = + data?.assetOrError && data.assetOrError.__typename === 'Asset' && data.assetOrError.definition; + return (
- {loading && ( + {loading ? ( - )} - - {/* {data?.assetOrError && data.assetOrError.__typename === 'Asset' && ( - - )} */} - {data?.assetOrError && - data.assetOrError.__typename === 'Asset' && - data.assetOrError.definition && ( - - )} + ) : navigatedDirectlyToTime ? ( + + + This is a historical view of materializations as of{' '} + + + + . + + } + description={ + setNavigatedDirectlyToTime(false)} underline="always"> + {definition + ? 'Show definition and latest materializations' + : 'Show latest materializations'} + + } + /> + + ) : definition ? ( + + ) : undefined}
- +
); }; diff --git a/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx b/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx index 589c6c8c6e07d..2ad966e8bfe43 100644 --- a/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx +++ b/js_modules/dagit/packages/core/src/assets/useMaterializationBuckets.tsx @@ -6,6 +6,7 @@ const NO_PARTITION_KEY = '__NO_PARTITION__'; export type HistoricalMaterialization = { latest: AssetMaterializationFragment; + timestamp: string; predecessors?: AssetMaterializationFragment[]; }; @@ -25,6 +26,7 @@ export const useMaterializationBuckets = (config: Config): HistoricalMaterializa if (!hasPartitions || !shouldBucketPartitions) { return materializations.map((materialization) => ({ latest: materialization, + timestamp: materialization.materializationEvent.timestamp, })); } @@ -44,7 +46,7 @@ export const useMaterializationBuckets = (config: Config): HistoricalMaterializa Number(b.materializationEvent?.timestamp) - Number(a.materializationEvent?.timestamp), ); const [latest, ...predecessors] = materializationsForKey; - return {latest, predecessors}; + return {latest, predecessors, timestamp: latest.materializationEvent.timestamp}; }; return Object.keys(buckets) diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index 92b36daefa473..4127f045f8ac4 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -156,7 +156,14 @@ const AssetGraphExplorerWithData: React.FC< 'replace', ); }, - [explorerPath, selectedGraphNode, graphData, onChangeExplorerPath, history], + [ + fetchAssetDefinitionLocation, + explorerPath, + onChangeExplorerPath, + history, + selectedGraphNode, + graphData, + ], ); const {all: highlighted} = React.useMemo( diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx index d6b8d9506bafc..d0a49193debdc 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx @@ -18,7 +18,7 @@ import {markdownToPlaintext} from '../../ui/Markdown'; import {MenuItemWIP, MenuWIP} from '../../ui/Menu'; import {FontFamily} from '../../ui/styles'; import {RepoAddress} from '../types'; -import {workspacePath} from '../workspacePath'; +import {workspacePath, workspacePipelinePathGuessRepo} from '../workspacePath'; import {assetKeyToString, Status} from './Utils'; import {AssetNodeFragment} from './types/AssetNodeFragment'; @@ -94,11 +94,19 @@ export const AssetNode: React.FC<{ }`} data-tooltip-style={RunLinkTooltipStyle} style={{overflow: 'hidden', textOverflow: 'ellipsis', paddingRight: 8}} - to={workspacePath( - repoAddress.name, - repoAddress.location, - `jobs/${runOrError.pipelineName}:${runOrError.mode}`, - )} + to={ + repoAddress.name + ? workspacePath( + repoAddress.name, + repoAddress.location, + `jobs/${runOrError.pipelineName}:${runOrError.mode}`, + ) + : workspacePipelinePathGuessRepo( + `${runOrError.pipelineName}:${runOrError.mode}`, + true, + '', + ) + } > {`${runOrError.pipelineName}${ runOrError.mode !== 'default' ? `:${runOrError.mode}` : '' diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index eed9ba807d934..2d4354b70e2ad 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -56,7 +56,13 @@ export const SidebarAssetInfo: React.FC<{ {node.assetMaterializations.length ? ( - + {}} + /> ) : null} From 4380e4b0489ccbae829e7bcb4af694eef360b276 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 23 Nov 2021 14:23:10 -0600 Subject: [PATCH 08/22] Rm tests no longer relevant --- .../core/src/assets/AssetDetails.test.tsx | 71 ------------------- .../assets/AssetLineageElements.stories.tsx | 6 +- .../core/src/assets/AssetView.test.tsx | 71 ------------------- 3 files changed, 3 insertions(+), 145 deletions(-) delete mode 100644 js_modules/dagit/packages/core/src/assets/AssetDetails.test.tsx delete mode 100644 js_modules/dagit/packages/core/src/assets/AssetView.test.tsx diff --git a/js_modules/dagit/packages/core/src/assets/AssetDetails.test.tsx b/js_modules/dagit/packages/core/src/assets/AssetDetails.test.tsx deleted file mode 100644 index 40c74f46b1b3c..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/AssetDetails.test.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import {MockList} from '@graphql-tools/mock'; -import {render, screen, waitFor} from '@testing-library/react'; -import * as React from 'react'; - -import {TestProvider} from '../testing/TestProvider'; -import {hyphenatedName} from '../testing/defaultMocks'; - -import {AssetView} from './AssetView'; - -describe('AssetView', () => { - const defaultMocks = { - EventMetadataEntry: () => ({ - __typename: 'EventAssetMetadataEntry', - label: hyphenatedName, - jsonString: () => '[]', - }), - }; - - describe('Historical materialization alert', () => { - it('does not show alert if no `asOf` is provided', async () => { - render( - - - , - ); - - await waitFor(() => { - expect(screen.queryByText(/this is a historical asset snapshot\./i)).toBeNull(); - }); - }); - - it('does not show alert if timestamps match', async () => { - const mocks = { - Asset: () => ({ - assetMaterializations: () => new MockList(1), - }), - StepMaterializationEvent: () => ({ - timestamp: () => '123', - }), - }; - render( - - - , - ); - - await waitFor(() => { - expect(screen.queryByText(/this is a historical asset snapshot\./i)).toBeNull(); - }); - }); - - it('does show alert if timestamps do not match', async () => { - let counter = 100; - const mocks = { - StepMaterializationEvent: () => ({ - timestamp: () => `${counter--}`, - }), - }; - - render( - - - , - ); - - await waitFor(() => { - expect(screen.queryByText(/this is a historical asset snapshot\./i)).toBeVisible(); - }); - }); - }); -}); diff --git a/js_modules/dagit/packages/core/src/assets/AssetLineageElements.stories.tsx b/js_modules/dagit/packages/core/src/assets/AssetLineageElements.stories.tsx index 10f0ad262c1bf..216d735e49781 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetLineageElements.stories.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetLineageElements.stories.tsx @@ -2,7 +2,7 @@ import {Meta} from '@storybook/react/types-6-0'; import * as React from 'react'; import {AssetLineageElements} from './AssetLineageElements'; -import {AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage} from './types/AssetQuery'; +import {AssetLineageFragment} from './types/AssetLineageFragment'; // eslint-disable-next-line import/no-default-export export default { @@ -12,7 +12,7 @@ export default { export const FewParents = () => { const timestamp = React.useMemo(() => Date.now(), []); - const elements: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage[] = [ + const elements: AssetLineageFragment[] = [ { __typename: 'AssetLineageInfo', partitions: ['2021-01-01'], @@ -44,7 +44,7 @@ export const FewParents = () => { export const ManyParents = () => { const timestamp = React.useMemo(() => Date.now(), []); - const elements: AssetQuery_assetOrError_Asset_assetMaterializations_materializationEvent_assetLineage[] = []; + const elements: AssetLineageFragment[] = []; for (let ii = 0; ii < 20; ii++) { elements.push({ __typename: 'AssetLineageInfo', diff --git a/js_modules/dagit/packages/core/src/assets/AssetView.test.tsx b/js_modules/dagit/packages/core/src/assets/AssetView.test.tsx deleted file mode 100644 index 40c74f46b1b3c..0000000000000 --- a/js_modules/dagit/packages/core/src/assets/AssetView.test.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import {MockList} from '@graphql-tools/mock'; -import {render, screen, waitFor} from '@testing-library/react'; -import * as React from 'react'; - -import {TestProvider} from '../testing/TestProvider'; -import {hyphenatedName} from '../testing/defaultMocks'; - -import {AssetView} from './AssetView'; - -describe('AssetView', () => { - const defaultMocks = { - EventMetadataEntry: () => ({ - __typename: 'EventAssetMetadataEntry', - label: hyphenatedName, - jsonString: () => '[]', - }), - }; - - describe('Historical materialization alert', () => { - it('does not show alert if no `asOf` is provided', async () => { - render( - - - , - ); - - await waitFor(() => { - expect(screen.queryByText(/this is a historical asset snapshot\./i)).toBeNull(); - }); - }); - - it('does not show alert if timestamps match', async () => { - const mocks = { - Asset: () => ({ - assetMaterializations: () => new MockList(1), - }), - StepMaterializationEvent: () => ({ - timestamp: () => '123', - }), - }; - render( - - - , - ); - - await waitFor(() => { - expect(screen.queryByText(/this is a historical asset snapshot\./i)).toBeNull(); - }); - }); - - it('does show alert if timestamps do not match', async () => { - let counter = 100; - const mocks = { - StepMaterializationEvent: () => ({ - timestamp: () => `${counter--}`, - }), - }; - - render( - - - , - ); - - await waitFor(() => { - expect(screen.queryByText(/this is a historical asset snapshot\./i)).toBeVisible(); - }); - }); - }); -}); From 16138ee575352172e47c2f6af77937d19bdc5bae Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 23 Nov 2021 15:20:36 -0600 Subject: [PATCH 09/22] Apply black --- .../dagster_graphql/schema/asset_graph.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py b/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py index cb6d2d37f37c7..ddfe48d8f1489 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py @@ -20,9 +20,7 @@ def __init__(self, external_repository, input_name, asset_key): self._external_repository = check.inst_param( external_repository, "external_repository", ExternalRepository ) - self._asset_key = check.inst_param( - asset_key, "asset_key", AssetKey - ) + self._asset_key = check.inst_param(asset_key, "asset_key", AssetKey) super().__init__(inputName=input_name) def resolve_asset(self, _graphene_info): @@ -80,14 +78,15 @@ def resolve_dependedBy(self, _graphene_info): for item in self._external_repository.get_external_asset_nodes(): for dep in item.dependencies: if dep.upstream_asset_key == self._external_asset_node.asset_key: - results.append(GrapheneAssetDependency( - external_repository=self._external_repository, - input_name=dep.input_name, - asset_key=item.asset_key, - )) + results.append( + GrapheneAssetDependency( + external_repository=self._external_repository, + input_name=dep.input_name, + asset_key=item.asset_key, + ) + ) return results - def resolve_assetMaterializations(self, graphene_info, **kwargs): from ..implementation.fetch_assets import get_asset_events From 4b25d48d46779841a8485a61b51fcd7d84b9c6ea Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 29 Nov 2021 22:32:08 -0500 Subject: [PATCH 10/22] Auto-refreshing materialization + in progress run overlays on Asset graph --- .../asset-graph/AssetGraphExplorer.tsx | 97 ++++- .../src/workspace/asset-graph/AssetNode.tsx | 335 ++++++++++-------- .../asset-graph/SidebarAssetInfo.tsx | 40 ++- .../core/src/workspace/asset-graph/Utils.tsx | 100 +++--- .../asset-graph/types/AssetGraphLiveQuery.ts | 207 +++++++++++ .../asset-graph/types/AssetGraphQuery.ts | 146 +------- .../asset-graph/types/AssetNodeFragment.ts | 146 -------- .../types/AssetNodeLiveFragment.ts | 160 +++++++++ 8 files changed, 726 insertions(+), 505 deletions(-) create mode 100644 js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphLiveQuery.ts create mode 100644 js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeLiveFragment.ts diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index 9450924fadbe4..0872cd91d8bd8 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -1,11 +1,11 @@ -import {gql, useQuery} from '@apollo/client'; +import {gql, QueryResult, useQuery} from '@apollo/client'; import {uniq, without} from 'lodash'; import React from 'react'; import {useHistory} from 'react-router-dom'; import styled from 'styled-components/macro'; import {filterByQuery} from '../../app/GraphQueryImpl'; -import {LATEST_MATERIALIZATION_METADATA_FRAGMENT} from '../../assets/LastMaterializationMetadata'; +import {QueryCountdown} from '../../app/QueryCountdown'; import {LaunchRootExecutionButton} from '../../execute/LaunchRootExecutionButton'; import {SVGViewport} from '../../graph/SVGViewport'; import {useDocumentTitle} from '../../hooks/useDocumentTitle'; @@ -21,17 +21,24 @@ import {repoAddressToSelector} from '../repoAddressToSelector'; import {RepoAddress} from '../types'; import {AssetLinks} from './AssetLinks'; -import {AssetNode, ASSET_NODE_FRAGMENT, getNodeDimensions} from './AssetNode'; +import { + AssetNode, + ASSET_NODE_FRAGMENT, + ASSET_NODE_LIVE_FRAGMENT, + getNodeDimensions, +} from './AssetNode'; import {ForeignNode, getForeignNodeDimensions} from './ForeignNode'; import {SidebarAssetInfo} from './SidebarAssetInfo'; import { - buildGraphComputeStatuses, buildGraphData, + buildLiveData, GraphData, graphHasCycles, layoutGraph, + LiveData, Node, } from './Utils'; +import {AssetGraphLiveQuery, AssetGraphLiveQueryVariables} from './types/AssetGraphLiveQuery'; import { AssetGraphQuery, AssetGraphQueryVariables, @@ -51,11 +58,25 @@ interface Props { export const AssetGraphExplorer: React.FC = (props) => { const {repoAddress, explorerPath} = props; const pipelineSelector = buildPipelineSelector(repoAddress || null, explorerPath.pipelineName); + const repositorySelector = { + repositoryLocationName: repoAddress.location, + repositoryName: repoAddress.name, + }; + const queryResult = useQuery(ASSETS_GRAPH_QUERY, { variables: {pipelineSelector}, notifyOnNetworkStatusChange: true, }); + const liveResult = useQuery( + ASSETS_GRAPH_LIVE_QUERY, + { + variables: {pipelineSelector, repositorySelector}, + notifyOnNetworkStatusChange: true, + pollInterval: 5 * 1000, + }, + ); + useDocumentTitle('Assets'); const graphData = React.useMemo(() => { @@ -65,6 +86,10 @@ export const AssetGraphExplorer: React.FC = (props) => { return buildGraphData(queryResult.data.pipelineOrError.assetNodes, explorerPath.pipelineName); }, [queryResult, explorerPath.pipelineName]); + const liveDataByNode = React.useMemo(() => { + return graphData ? buildLiveData(graphData, liveResult.data) : {}; + }, [graphData, liveResult]); + return ( {({pipelineOrError}) => { @@ -93,14 +118,25 @@ export const AssetGraphExplorer: React.FC = (props) => { ); } - return ; + return ( + + ); }} ); }; const AssetGraphExplorerWithData: React.FC< - {graphData: ReturnType} & Props + { + graphData: ReturnType; + liveDataByNode: LiveData; + liveDataQueryResult: QueryResult; + } & Props > = (props) => { const { repoAddress, @@ -108,6 +144,8 @@ const AssetGraphExplorerWithData: React.FC< selectedHandle, explorerPath, onChangeExplorerPath, + liveDataQueryResult, + liveDataByNode, graphData, } = props; @@ -170,8 +208,6 @@ const AssetGraphExplorerWithData: React.FC< ); const layout = React.useMemo(() => layoutGraph(graphData), [graphData]); - const computeStatuses = React.useMemo(() => buildGraphComputeStatuses(graphData), [graphData]); - console.log(layout); return ( h.handleID === graphNode.definition.opName)!.solid .definition.metadata } selected={selectedGraphNode === graphNode} - computeStatus={computeStatuses[graphNode.id]} repoAddress={repoAddress} secondaryHighlight={ explorerPath.opsQuery @@ -258,6 +294,9 @@ const AssetGraphExplorerWithData: React.FC< )} +
+ +
h.solid)} @@ -288,6 +327,7 @@ const AssetGraphExplorerWithData: React.FC< selectedGraphNode && selectedDefinition ? ( @@ -299,17 +339,47 @@ const AssetGraphExplorerWithData: React.FC< ); }; +const ASSETS_GRAPH_LIVE_QUERY = gql` + query AssetGraphLiveQuery( + $pipelineSelector: PipelineSelector! + $repositorySelector: RepositorySelector! + ) { + repositoryOrError(repositorySelector: $repositorySelector) { + __typename + ... on Repository { + id + name + inProgressRunsByStep { + stepKey + runs { + id + runId + } + } + } + } + pipelineOrError(params: $pipelineSelector) { + ... on Pipeline { + id + assetNodes { + id + ...AssetNodeLiveFragment + } + } + } + } + ${ASSET_NODE_LIVE_FRAGMENT} +`; + const ASSETS_GRAPH_QUERY = gql` query AssetGraphQuery($pipelineSelector: PipelineSelector!) { pipelineOrError(params: $pipelineSelector) { ... on Pipeline { id assetNodes { - ...AssetNodeFragment id - assetKey { - path - } + ...AssetNodeFragment + dependencies { inputName upstreamAsset { @@ -324,7 +394,6 @@ const ASSETS_GRAPH_QUERY = gql` } } ${ASSET_NODE_FRAGMENT} - ${LATEST_MATERIALIZATION_METADATA_FRAGMENT} `; const SVGContainer = styled.svg` diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx index 7f7a6d78b7cbe..ac4e290ace783 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetNode.tsx @@ -23,186 +23,194 @@ import {ColorsWIP} from '../../ui/Colors'; import {IconWIP} from '../../ui/Icon'; import {markdownToPlaintext} from '../../ui/Markdown'; import {MenuItemWIP, MenuWIP} from '../../ui/Menu'; +import {Spinner} from '../../ui/Spinner'; +import {Tooltip} from '../../ui/Tooltip'; import {FontFamily} from '../../ui/styles'; import {repoAddressToSelector} from '../repoAddressToSelector'; import {RepoAddress} from '../types'; import {workspacePath} from '../workspacePath'; -import {assetKeyToString, Status} from './Utils'; +import {assetKeyToString, LiveDataForNode} from './Utils'; import {AssetNodeFragment} from './types/AssetNodeFragment'; export const AssetNode: React.FC<{ definition: AssetNodeFragment; + liveData?: LiveDataForNode; metadata: {key: string; value: string}[]; selected: boolean; - computeStatus: Status; repoAddress: RepoAddress; secondaryHighlight: boolean; -}> = React.memo( - ({definition, metadata, selected, computeStatus, repoAddress, secondaryHighlight}) => { - const [launchPipelineExecution] = useMutation( - LAUNCH_PIPELINE_EXECUTION_MUTATION, - ); - const {basePath} = React.useContext(AppContext); - const {materializationEvent: event, runOrError} = definition.assetMaterializations[0] || {}; - const kind = metadata.find((m) => m.key === 'kind')?.value; +}> = React.memo(({definition, metadata, selected, repoAddress, secondaryHighlight, liveData}) => { + const [launchPipelineExecution] = useMutation( + LAUNCH_PIPELINE_EXECUTION_MUTATION, + ); + const {basePath} = React.useContext(AppContext); + const {materializationEvent: event, runOrError} = liveData?.lastMaterialization || {}; + const kind = metadata.find((m) => m.key === 'kind')?.value; - const onLaunch = async () => { - if (!definition.jobName) { - return; - } + const onLaunch = async () => { + if (!definition.jobName) { + return; + } - try { - const result = await launchPipelineExecution({ - variables: { - executionParams: { - selector: { - pipelineName: definition.jobName, - ...repoAddressToSelector(repoAddress), - }, - mode: 'default', - stepKeys: [definition.opName], + try { + const result = await launchPipelineExecution({ + variables: { + executionParams: { + selector: { + pipelineName: definition.jobName, + ...repoAddressToSelector(repoAddress), }, + mode: 'default', + stepKeys: [definition.opName], }, - }); - handleLaunchResult(basePath, definition.jobName, result, true); - } catch (error) { - showLaunchError(error as Error); - } - }; + }, + }); + handleLaunchResult(basePath, definition.jobName, result, true); + } catch (error) { + showLaunchError(error as Error); + } + }; - return ( - - - Launch run to build{' '} - - {assetKeyToString(definition.assetKey)} - + return ( + + + Launch run to build{' '} + + {assetKeyToString(definition.assetKey)} - } - icon="open_in_new" - onClick={onLaunch} - /> - - } - > - - - - - {assetKeyToString(definition.assetKey)} -
- {computeStatus === 'old' && ( - - upstream -
- changed - - )} - - {definition.description && ( - - {markdownToPlaintext(definition.description).split('\n')[0]} - + + } + icon="open_in_new" + onClick={onLaunch} + /> + + } + > + + + + + {assetKeyToString(definition.assetKey)} +
+ {liveData && liveData.inProgressRunIds.length > 0 && ( + + Run ID:{' '} + {liveData.inProgressRunIds.map((runId) => ( + + {runId} + + ))} +
+ } + > + + )} - {event ? ( - - {runOrError.__typename === 'Run' && ( - - - {`${runOrError.pipelineName}${ - runOrError.mode !== 'default' ? `:${runOrError.mode}` : '' - }`} - - - {titleForRun({runId: runOrError.runId})} - - - )} - + {liveData?.computeStatus === 'old' && ( + + upstream +
+ changed + + )} + + {definition.description && ( + {markdownToPlaintext(definition.description).split('\n')[0]} + )} + {event ? ( + + {runOrError?.__typename === 'Run' && ( - {event.stepStats.endTime ? ( - - ) : ( - 'Never' - )} - - - - ) : ( - - - No materializations - — + + {`${runOrError.pipelineName}${ + runOrError.mode !== 'default' ? `:${runOrError.mode}` : '' + }`} + + + {titleForRun({runId: runOrError.runId})} + - - — - — - - - )} - {kind && ( - { - window.requestAnimationFrame(() => - document.dispatchEvent(new Event('show-kind-info')), - ); - }, + )} + + + {event.stepStats.endTime ? ( + + ) : ( + 'Never' + )} + + + + ) : ( + + + No materializations + — + + + — + — + + + )} + {kind && ( + { + window.requestAnimationFrame(() => + document.dispatchEvent(new Event('show-kind-info')), + ); }, - ]} - /> - )} - - - - ); - }, - isEqual, -); + }, + ]} + /> + )} + + + + ); +}, isEqual); -export const ASSET_NODE_FRAGMENT = gql` - fragment AssetNodeFragment on AssetNode { +export const ASSET_NODE_LIVE_FRAGMENT = gql` + fragment AssetNodeLiveFragment on AssetNode { id opName - description - jobName - assetKey { - path - } assetMaterializations(limit: 1) { ...LatestMaterializationMetadataFragment @@ -235,7 +243,22 @@ export const ASSET_NODE_FRAGMENT = gql` ${METADATA_ENTRY_FRAGMENT} `; -export const getNodeDimensions = (def: AssetNodeFragment) => { +export const ASSET_NODE_FRAGMENT = gql` + fragment AssetNodeFragment on AssetNode { + id + opName + description + jobName + assetKey { + path + } + } +`; + +export const getNodeDimensions = (def: { + assetKey: {path: string[]}; + description?: string | null; +}) => { let height = 95; if (def.description) { height += 25; @@ -283,10 +306,12 @@ const AssetNodeBox = styled.div` const Name = styled.div` display: flex; padding: 4px 6px; + align-items: center; font-family: ${FontFamily.monospace}; border-top-left-radius: 5px; border-top-right-radius: 5px; font-weight: 600; + gap: 4px; `; const Description = styled.div` diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index eed9ba807d934..42c9e0dd12fb5 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -8,20 +8,24 @@ import {Description} from '../../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../../pipelines/SidebarComponents'; import {GraphExplorerSolidHandleFragment_solid_definition} from '../../pipelines/types/GraphExplorerSolidHandleFragment'; import {pluginForMetadata} from '../../plugins'; +import {BaseTag} from '../../ui/BaseTag'; import {Box} from '../../ui/Box'; import {ColorsWIP} from '../../ui/Colors'; import {IconWIP} from '../../ui/Icon'; +import {Spinner} from '../../ui/Spinner'; import {RepoAddress} from '../types'; -import {assetKeyToString} from './Utils'; +import {assetKeyToString, LiveDataForNode} from './Utils'; import {AssetGraphQuery_pipelineOrError_Pipeline_assetNodes} from './types/AssetGraphQuery'; export const SidebarAssetInfo: React.FC<{ definition: GraphExplorerSolidHandleFragment_solid_definition; node: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes; + liveData: LiveDataForNode; repoAddress: RepoAddress; -}> = ({node, definition, repoAddress}) => { +}> = ({node, definition, repoAddress, liveData}) => { const Plugin = pluginForMetadata(definition.metadata); + const {lastMaterialization, inProgressRunIds} = liveData || {}; return (
@@ -36,11 +40,37 @@ export const SidebarAssetInfo: React.FC<{
+ {inProgressRunIds?.length > 0 && ( + + This asset may be materialized soon by + + {['12341514'].map((runId) => ( + + + + {`Run: ${runId.slice(0, 8)}`} + + } + /> + + ))} + + )} - {node.assetMaterializations.length ? ( + {lastMaterialization ? ( <>
- +
@@ -54,7 +84,7 @@ export const SidebarAssetInfo: React.FC<{ )} - {node.assetMaterializations.length ? ( + {lastMaterialization ? ( diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx index 00a50bff2fdff..10bf8c979616f 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/Utils.tsx @@ -3,6 +3,10 @@ import * as dagre from 'dagre'; import {getNodeDimensions} from './AssetNode'; import {getForeignNodeDimensions} from './ForeignNode'; +import { + AssetGraphLiveQuery, + AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations, +} from './types/AssetGraphLiveQuery'; import { AssetGraphQuery_pipelineOrError_Pipeline_assetNodes, AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetKey, @@ -43,8 +47,8 @@ export function assetKeyToString(key: {path: string[]}) { export const buildGraphData = (assetNodes: AssetNode[], jobName?: string) => { const nodes: {[id: string]: Node} = {}; - const downstream: {[downstreamId: string]: {[upstreamId: string]: string}} = {}; - const upstream: {[upstreamId: string]: {[downstreamId: string]: boolean}} = {}; + const downstream: {[id: string]: {[childAssetId: string]: string}} = {}; + const upstream: {[id: string]: {[parentAssetId: string]: boolean}} = {}; assetNodes.forEach((definition: AssetNode) => { const assetKeyJson = JSON.stringify(definition.assetKey.path); @@ -172,57 +176,73 @@ export const buildSVGPath = pathVerticalDiagonal({ y: (s: any) => s.y, }); -export function buildGraphComputeStatuses(graphData: GraphData) { - const timestamps: {[key: string]: number} = {}; - for (const node of Object.values(graphData.nodes)) { - timestamps[node.id] = - node.definition.assetMaterializations[0]?.materializationEvent.stepStats?.startTime || 0; +export type Status = 'good' | 'old' | 'none' | 'unknown'; + +export interface LiveDataForNode { + computeStatus: Status; + inProgressRunIds: string[]; + lastMaterialization: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations; + lastStepStart: number; +} +export interface LiveData { + [assetId: string]: LiveDataForNode; +} + +export const buildLiveData = ( + graph: ReturnType, + queryResult?: AssetGraphLiveQuery, +) => { + const data: LiveData = {}; + + if (!queryResult) { + return data; } - const upstream: {[key: string]: string[]} = {}; - Object.keys(graphData.downstream).forEach((upstreamId) => { - const downstreamIds = Object.keys(graphData.downstream[upstreamId]); - downstreamIds.forEach((downstreamId) => { - upstream[downstreamId] = upstream[downstreamId] || []; - upstream[downstreamId].push(upstreamId); - }); - }); + const {repositoryOrError, pipelineOrError} = queryResult; - const statuses: {[key: string]: Status} = {}; + const nodes = pipelineOrError.__typename === 'Pipeline' ? pipelineOrError.assetNodes : []; + const inProgressRunsByStep = + repositoryOrError.__typename === 'Repository' ? repositoryOrError.inProgressRunsByStep : []; - for (const asset of Object.values(graphData.nodes)) { - if (asset.definition.assetMaterializations.length === 0) { - statuses[asset.id] = 'none'; - } + for (const node of nodes) { + const lastMaterialization = node.assetMaterializations[0]; + const lastStepStart = lastMaterialization?.materializationEvent.stepStats?.startTime || 0; + + data[node.id] = { + lastStepStart, + lastMaterialization, + inProgressRunIds: + inProgressRunsByStep.find((r) => r.stepKey === node.opName)?.runs.map((r) => r.runId) || [], + computeStatus: lastMaterialization ? 'unknown' : 'none', + }; } - for (const asset of Object.values(graphData.nodes)) { - const id = JSON.stringify(asset.assetKey.path); - statuses[id] = findComputeStatusForId(timestamps, statuses, upstream, id); + + for (const asset of nodes) { + data[asset.id].computeStatus = findComputeStatusForId(data, graph.upstream, asset.id); } - return statuses; -} -export type Status = 'good' | 'old' | 'none'; + return data; +}; function findComputeStatusForId( - timestamps: {[key: string]: number}, - statuses: {[key: string]: Status}, - upstream: {[key: string]: string[]}, - id: string, + data: LiveData, + upstream: {[assetId: string]: {[upstreamAssetId: string]: boolean}}, + assetId: string, ): Status { - const ts = timestamps[id]; - const upstreamIds = upstream[id] || []; - if (id in statuses) { - return statuses[id]; + if (!data[assetId]) { + // Currently compute status assumes foreign nodes are up to date + // and only shows "upstream changed" for upstreams in the same job + return 'good'; + } + const ts = data[assetId].lastStepStart; + const upstreamIds = Object.keys(upstream[assetId] || {}); + if (data[assetId].computeStatus !== 'unknown') { + return data[assetId].computeStatus; } - statuses[id] = upstreamIds.some((uid) => timestamps[uid] > ts) + return upstreamIds.some((uid) => data[uid]?.lastStepStart > ts) ? 'old' - : upstreamIds.some( - (uid) => findComputeStatusForId(timestamps, statuses, upstream, uid) !== 'good', - ) + : upstreamIds.some((uid) => findComputeStatusForId(data, upstream, uid) !== 'good') ? 'old' : 'good'; - - return statuses[id]; } diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphLiveQuery.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphLiveQuery.ts new file mode 100644 index 0000000000000..0862cfaa0ed5e --- /dev/null +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphLiveQuery.ts @@ -0,0 +1,207 @@ +/* tslint:disable */ +/* eslint-disable */ +// @generated +// This file was automatically generated and should not be edited. + +import { PipelineSelector, RepositorySelector, RunStatus } from "./../../../types/globalTypes"; + +// ==================================================== +// GraphQL query operation: AssetGraphLiveQuery +// ==================================================== + +export interface AssetGraphLiveQuery_repositoryOrError_PythonError { + __typename: "PythonError" | "RepositoryNotFoundError"; +} + +export interface AssetGraphLiveQuery_repositoryOrError_Repository_inProgressRunsByStep_runs { + __typename: "Run"; + id: string; + runId: string; +} + +export interface AssetGraphLiveQuery_repositoryOrError_Repository_inProgressRunsByStep { + __typename: "InProgressRunsByStep"; + stepKey: string; + runs: AssetGraphLiveQuery_repositoryOrError_Repository_inProgressRunsByStep_runs[]; +} + +export interface AssetGraphLiveQuery_repositoryOrError_Repository { + __typename: "Repository"; + id: string; + name: string; + inProgressRunsByStep: AssetGraphLiveQuery_repositoryOrError_Repository_inProgressRunsByStep[]; +} + +export type AssetGraphLiveQuery_repositoryOrError = AssetGraphLiveQuery_repositoryOrError_PythonError | AssetGraphLiveQuery_repositoryOrError_Repository; + +export interface AssetGraphLiveQuery_pipelineOrError_PipelineNotFoundError { + __typename: "PipelineNotFoundError" | "InvalidSubsetError" | "PythonError"; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError = AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_RunNotFoundError | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run; + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_stepStats; + materialization: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError; + materializationEvent: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes { + __typename: "AssetNode"; + id: string; + opName: string | null; + assetMaterializations: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations[]; +} + +export interface AssetGraphLiveQuery_pipelineOrError_Pipeline { + __typename: "Pipeline"; + id: string; + assetNodes: AssetGraphLiveQuery_pipelineOrError_Pipeline_assetNodes[]; +} + +export type AssetGraphLiveQuery_pipelineOrError = AssetGraphLiveQuery_pipelineOrError_PipelineNotFoundError | AssetGraphLiveQuery_pipelineOrError_Pipeline; + +export interface AssetGraphLiveQuery { + repositoryOrError: AssetGraphLiveQuery_repositoryOrError; + pipelineOrError: AssetGraphLiveQuery_pipelineOrError; +} + +export interface AssetGraphLiveQueryVariables { + pipelineSelector: PipelineSelector; + repositorySelector: RepositorySelector; +} diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts index 63ca9bb77443e..10b8236c1076c 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetGraphQuery.ts @@ -3,7 +3,7 @@ // @generated // This file was automatically generated and should not be edited. -import { PipelineSelector, RunStatus } from "./../../../types/globalTypes"; +import { PipelineSelector } from "./../../../types/globalTypes"; // ==================================================== // GraphQL query operation: AssetGraphQuery @@ -18,149 +18,6 @@ export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetKey { path: string[]; } -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_RunNotFoundError { - __typename: "RunNotFoundError" | "PythonError"; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run_repositoryOrigin { - __typename: "RepositoryOrigin"; - id: string; - repositoryName: string; - repositoryLocationName: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run { - __typename: "Run"; - id: string; - runId: string; - mode: string; - pipelineName: string; - pipelineSnapshotId: string | null; - repositoryOrigin: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run_repositoryOrigin | null; - status: RunStatus; -} - -export type AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError = AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_RunNotFoundError | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError_Run; - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_stepStats { - __typename: "RunStepStats"; - endTime: number | null; - startTime: number | null; - stepKey: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { - __typename: "EventPathMetadataEntry"; - label: string; - description: string | null; - path: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { - __typename: "EventJsonMetadataEntry"; - label: string; - description: string | null; - jsonString: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { - __typename: "EventUrlMetadataEntry"; - label: string; - description: string | null; - url: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { - __typename: "EventTextMetadataEntry"; - label: string; - description: string | null; - text: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { - __typename: "EventMarkdownMetadataEntry"; - label: string; - description: string | null; - mdStr: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { - __typename: "EventPythonArtifactMetadataEntry"; - label: string; - description: string | null; - module: string; - name: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { - __typename: "EventFloatMetadataEntry"; - label: string; - description: string | null; - floatValue: number | null; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { - __typename: "EventIntMetadataEntry"; - label: string; - description: string | null; - intValue: number | null; - intRepr: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { - __typename: "EventPipelineRunMetadataEntry"; - label: string; - description: string | null; - runId: string; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { - __typename: "AssetKey"; - path: string[]; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { - __typename: "EventAssetMetadataEntry"; - label: string; - description: string | null; - assetKey: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; -} - -export type AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization { - __typename: "Materialization"; - metadataEntries: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization_metadataEntries[]; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage_assetKey { - __typename: "AssetKey"; - path: string[]; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage { - __typename: "AssetLineageInfo"; - assetKey: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage_assetKey; - partitions: string[]; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent { - __typename: "StepMaterializationEvent"; - runId: string; - timestamp: string; - stepKey: string | null; - stepStats: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_stepStats; - materialization: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_materialization; - assetLineage: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent_assetLineage[]; -} - -export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations { - __typename: "AssetMaterialization"; - partition: string | null; - runOrError: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_runOrError; - materializationEvent: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations_materializationEvent; -} - export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_dependencies_upstreamAsset_assetKey { __typename: "AssetKey"; path: string[]; @@ -185,7 +42,6 @@ export interface AssetGraphQuery_pipelineOrError_Pipeline_assetNodes { description: string | null; jobName: string | null; assetKey: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetKey; - assetMaterializations: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_assetMaterializations[]; dependencies: AssetGraphQuery_pipelineOrError_Pipeline_assetNodes_dependencies[]; } diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts index 8f19ca5ab5a91..653aba12db6b6 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeFragment.ts @@ -3,8 +3,6 @@ // @generated // This file was automatically generated and should not be edited. -import { RunStatus } from "./../../../types/globalTypes"; - // ==================================================== // GraphQL fragment: AssetNodeFragment // ==================================================== @@ -14,149 +12,6 @@ export interface AssetNodeFragment_assetKey { path: string[]; } -export interface AssetNodeFragment_assetMaterializations_runOrError_RunNotFoundError { - __typename: "RunNotFoundError" | "PythonError"; -} - -export interface AssetNodeFragment_assetMaterializations_runOrError_Run_repositoryOrigin { - __typename: "RepositoryOrigin"; - id: string; - repositoryName: string; - repositoryLocationName: string; -} - -export interface AssetNodeFragment_assetMaterializations_runOrError_Run { - __typename: "Run"; - id: string; - runId: string; - mode: string; - pipelineName: string; - pipelineSnapshotId: string | null; - repositoryOrigin: AssetNodeFragment_assetMaterializations_runOrError_Run_repositoryOrigin | null; - status: RunStatus; -} - -export type AssetNodeFragment_assetMaterializations_runOrError = AssetNodeFragment_assetMaterializations_runOrError_RunNotFoundError | AssetNodeFragment_assetMaterializations_runOrError_Run; - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_stepStats { - __typename: "RunStepStats"; - endTime: number | null; - startTime: number | null; - stepKey: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { - __typename: "EventPathMetadataEntry"; - label: string; - description: string | null; - path: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { - __typename: "EventJsonMetadataEntry"; - label: string; - description: string | null; - jsonString: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { - __typename: "EventUrlMetadataEntry"; - label: string; - description: string | null; - url: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { - __typename: "EventTextMetadataEntry"; - label: string; - description: string | null; - text: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { - __typename: "EventMarkdownMetadataEntry"; - label: string; - description: string | null; - mdStr: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { - __typename: "EventPythonArtifactMetadataEntry"; - label: string; - description: string | null; - module: string; - name: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { - __typename: "EventFloatMetadataEntry"; - label: string; - description: string | null; - floatValue: number | null; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { - __typename: "EventIntMetadataEntry"; - label: string; - description: string | null; - intValue: number | null; - intRepr: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { - __typename: "EventPipelineRunMetadataEntry"; - label: string; - description: string | null; - runId: string; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { - __typename: "AssetKey"; - path: string[]; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { - __typename: "EventAssetMetadataEntry"; - label: string; - description: string | null; - assetKey: AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; -} - -export type AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_materialization { - __typename: "Materialization"; - metadataEntries: AssetNodeFragment_assetMaterializations_materializationEvent_materialization_metadataEntries[]; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage_assetKey { - __typename: "AssetKey"; - path: string[]; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage { - __typename: "AssetLineageInfo"; - assetKey: AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage_assetKey; - partitions: string[]; -} - -export interface AssetNodeFragment_assetMaterializations_materializationEvent { - __typename: "StepMaterializationEvent"; - runId: string; - timestamp: string; - stepKey: string | null; - stepStats: AssetNodeFragment_assetMaterializations_materializationEvent_stepStats; - materialization: AssetNodeFragment_assetMaterializations_materializationEvent_materialization; - assetLineage: AssetNodeFragment_assetMaterializations_materializationEvent_assetLineage[]; -} - -export interface AssetNodeFragment_assetMaterializations { - __typename: "AssetMaterialization"; - partition: string | null; - runOrError: AssetNodeFragment_assetMaterializations_runOrError; - materializationEvent: AssetNodeFragment_assetMaterializations_materializationEvent; -} - export interface AssetNodeFragment { __typename: "AssetNode"; id: string; @@ -164,5 +19,4 @@ export interface AssetNodeFragment { description: string | null; jobName: string | null; assetKey: AssetNodeFragment_assetKey; - assetMaterializations: AssetNodeFragment_assetMaterializations[]; } diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeLiveFragment.ts b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeLiveFragment.ts new file mode 100644 index 0000000000000..315de7c3690d8 --- /dev/null +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/types/AssetNodeLiveFragment.ts @@ -0,0 +1,160 @@ +/* tslint:disable */ +/* eslint-disable */ +// @generated +// This file was automatically generated and should not be edited. + +import { RunStatus } from "./../../../types/globalTypes"; + +// ==================================================== +// GraphQL fragment: AssetNodeLiveFragment +// ==================================================== + +export interface AssetNodeLiveFragment_assetMaterializations_runOrError_RunNotFoundError { + __typename: "RunNotFoundError" | "PythonError"; +} + +export interface AssetNodeLiveFragment_assetMaterializations_runOrError_Run_repositoryOrigin { + __typename: "RepositoryOrigin"; + id: string; + repositoryName: string; + repositoryLocationName: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_runOrError_Run { + __typename: "Run"; + id: string; + runId: string; + mode: string; + pipelineName: string; + pipelineSnapshotId: string | null; + repositoryOrigin: AssetNodeLiveFragment_assetMaterializations_runOrError_Run_repositoryOrigin | null; + status: RunStatus; +} + +export type AssetNodeLiveFragment_assetMaterializations_runOrError = AssetNodeLiveFragment_assetMaterializations_runOrError_RunNotFoundError | AssetNodeLiveFragment_assetMaterializations_runOrError_Run; + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_stepStats { + __typename: "RunStepStats"; + endTime: number | null; + startTime: number | null; + stepKey: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry { + __typename: "EventPathMetadataEntry"; + label: string; + description: string | null; + path: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry { + __typename: "EventJsonMetadataEntry"; + label: string; + description: string | null; + jsonString: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry { + __typename: "EventUrlMetadataEntry"; + label: string; + description: string | null; + url: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry { + __typename: "EventTextMetadataEntry"; + label: string; + description: string | null; + text: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry { + __typename: "EventMarkdownMetadataEntry"; + label: string; + description: string | null; + mdStr: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry { + __typename: "EventPythonArtifactMetadataEntry"; + label: string; + description: string | null; + module: string; + name: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry { + __typename: "EventFloatMetadataEntry"; + label: string; + description: string | null; + floatValue: number | null; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry { + __typename: "EventIntMetadataEntry"; + label: string; + description: string | null; + intValue: number | null; + intRepr: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry { + __typename: "EventPipelineRunMetadataEntry"; + label: string; + description: string | null; + runId: string; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry { + __typename: "EventAssetMetadataEntry"; + label: string; + description: string | null; + assetKey: AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry_assetKey; +} + +export type AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries = AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPathMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventJsonMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventUrlMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventTextMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventMarkdownMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPythonArtifactMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventFloatMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventIntMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventPipelineRunMetadataEntry | AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries_EventAssetMetadataEntry; + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization { + __typename: "Materialization"; + metadataEntries: AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization_metadataEntries[]; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_assetLineage_assetKey { + __typename: "AssetKey"; + path: string[]; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent_assetLineage { + __typename: "AssetLineageInfo"; + assetKey: AssetNodeLiveFragment_assetMaterializations_materializationEvent_assetLineage_assetKey; + partitions: string[]; +} + +export interface AssetNodeLiveFragment_assetMaterializations_materializationEvent { + __typename: "StepMaterializationEvent"; + runId: string; + timestamp: string; + stepKey: string | null; + stepStats: AssetNodeLiveFragment_assetMaterializations_materializationEvent_stepStats; + materialization: AssetNodeLiveFragment_assetMaterializations_materializationEvent_materialization; + assetLineage: AssetNodeLiveFragment_assetMaterializations_materializationEvent_assetLineage[]; +} + +export interface AssetNodeLiveFragment_assetMaterializations { + __typename: "AssetMaterialization"; + partition: string | null; + runOrError: AssetNodeLiveFragment_assetMaterializations_runOrError; + materializationEvent: AssetNodeLiveFragment_assetMaterializations_materializationEvent; +} + +export interface AssetNodeLiveFragment { + __typename: "AssetNode"; + id: string; + opName: string | null; + assetMaterializations: AssetNodeLiveFragment_assetMaterializations[]; +} From 3f269d53201d533cc171f0713711384ee6c91a54 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 10:41:25 -0600 Subject: [PATCH 11/22] Address PR feedback --- .../asset-graph/SidebarAssetInfo.tsx | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx index 42c9e0dd12fb5..bb0629f9484f1 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/SidebarAssetInfo.tsx @@ -50,19 +50,18 @@ export const SidebarAssetInfo: React.FC<{ > This asset may be materialized soon by - {['12341514'].map((runId) => ( - - - - {`Run: ${runId.slice(0, 8)}`} - - } - /> - + {inProgressRunIds.map((runId) => ( + + + {`Run: ${runId.slice(0, 8)}`} + + } + /> ))} )} From 9c630a2d555ba379a32e79f35595a8bc1d7ae62f Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 11:36:22 -0600 Subject: [PATCH 12/22] Improve the algorithm for shift-selection, ignore foreign nodes and select all paths from A=>B --- .../asset-graph/AssetGraphExplorer.tsx | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx index 4127f045f8ac4..6ec53f23deb22 100644 --- a/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagit/packages/core/src/workspace/asset-graph/AssetGraphExplorer.tsx @@ -354,6 +354,22 @@ const AssetQueryInputContainer = styled.div` display: flex; `; +const graphDirectionOf = ({graph, from, to}: {graph: GraphData; from: Node; to: Node}) => { + const stack = [from]; + while (stack.length) { + const node = stack.pop()!; + + const downstream = [...Object.keys(graph.downstream[node.id] || {})] + .map((n) => graph.nodes[n]) + .filter(Boolean); + if (downstream.some((d) => d.id === to.id)) { + return 'downstream'; + } + stack.push(...downstream); + } + return 'upstream'; +}; + const opsInRange = ( {graph, from, to}: {graph: GraphData; from: Node; to: Node}, seen: string[] = [], @@ -364,21 +380,25 @@ const opsInRange = ( if (from.id === to.id) { return [to.definition.opName!]; } - const adjacent = [ - ...Object.keys(graph.upstream[from.id] || {}), - ...Object.keys(graph.downstream[from.id] || {}), - ].map((n) => graph.nodes[n]); - let best: string[] = []; + if (seen.length === 0 && graphDirectionOf({graph, from, to}) === 'upstream') { + [from, to] = [to, from]; + } + + const downstream = [...Object.keys(graph.downstream[from.id] || {})] + .map((n) => graph.nodes[n]) + .filter(Boolean); + + const ledToTarget: string[] = []; - for (const node of adjacent) { + for (const node of downstream) { if (seen.includes(node.id)) { continue; } const result: string[] = opsInRange({graph, from: node, to}, [...seen, from.id]); - if (result.length && (best.length === 0 || result.length < best.length)) { - best = [from.definition.opName!, ...result]; + if (result.length) { + ledToTarget.push(from.definition.opName!, ...result); } } - return best; + return uniq(ledToTarget); }; From d353e705b306fb6f35c317d1b4412f7528193cf1 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 11:40:41 -0600 Subject: [PATCH 13/22] Switch definition and upstream/downstream in the asset details panel --- .../core/src/assets/AssetNodeDefinition.tsx | 43 +++++++++++-------- .../core/src/pipelines/Description.tsx | 8 ++-- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx index 6fa6086c02fdd..1b3cba2157712 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx @@ -25,23 +25,7 @@ export const AssetNodeDefinition: React.FC<{assetNode: AssetNodeDefinitionFragme flex={{direction: 'row'}} border={{side: 'bottom', width: 4, color: ColorsWIP.KeylineGray}} > - - - Parent Assets ({assetNode.dependencies.length}) - - - - Child Assets ({assetNode.dependedBy.length}) - - - - + - + + + + Upstream Assets ({assetNode.dependencies.length}) + + + + Downstream Assets ({assetNode.dependedBy.length}) + + + ); }; diff --git a/js_modules/dagit/packages/core/src/pipelines/Description.tsx b/js_modules/dagit/packages/core/src/pipelines/Description.tsx index 9447a86c26331..7aa530659cbbf 100644 --- a/js_modules/dagit/packages/core/src/pipelines/Description.tsx +++ b/js_modules/dagit/packages/core/src/pipelines/Description.tsx @@ -5,6 +5,7 @@ import {Markdown} from '../ui/Markdown'; interface IDescriptionProps { description: string | null; + maxHeight?: number; } interface IDescriptionState { @@ -12,7 +13,7 @@ interface IDescriptionState { expanded: boolean; } -const MaxHeight = 320; +const DEFAULT_MAX_HEIGHT = 320; /* If `input` begins with whitespace and every line contains at least that whitespace, @@ -52,7 +53,8 @@ export class Description extends React.Component MaxHeight; + const hasMore = + this._container.current.clientHeight > (this.props.maxHeight || DEFAULT_MAX_HEIGHT); if (hasMore !== this.state.hasMore) { this.setState({hasMore}); } @@ -77,7 +79,7 @@ export class Description extends React.Component {!expanded && hasMore && } From 8b1816309c34c721badc15b45175e0f6cd07900c Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 12:11:38 -0600 Subject: [PATCH 14/22] Shrink nodes a bit --- .../packages/core/src/assets/AssetNodeDefinition.tsx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx index 1b3cba2157712..17ac8c8a232f6 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx @@ -7,11 +7,7 @@ import {PipelineReference} from '../pipelines/PipelineReference'; import {Box} from '../ui/Box'; import {ColorsWIP} from '../ui/Colors'; import {Subheading} from '../ui/Text'; -import { - AssetNode, - ASSET_NODE_FRAGMENT, - getNodeDimensions, -} from '../workspace/asset-graph/AssetNode'; +import {AssetNode, ASSET_NODE_FRAGMENT} from '../workspace/asset-graph/AssetNode'; import {assetKeyToString} from '../workspace/asset-graph/Utils'; import {AssetNodeFragment} from '../workspace/asset-graph/types/AssetNodeFragment'; @@ -109,7 +105,8 @@ const AssetList: React.FC<{ position: 'relative', display: 'inline-block', verticalAlign: 'top', - ...getNodeDimensions({...dep.asset, description: ''}), + height: 95, + width: 215, }} key={assetKeyToString(dep.asset.assetKey)} onClick={(e) => { From 5feaa7d8cef777e31abeaebab46d89817aec6b01 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 30 Nov 2021 16:06:11 -0600 Subject: [PATCH 15/22] Thread inProgressRunIds into the asset view --- .../src/assets/AssetMaterializationTable.tsx | 34 ++-- .../core/src/assets/AssetMaterializations.tsx | 6 + .../core/src/assets/AssetNeighborsGraph.tsx | 24 +-- .../core/src/assets/AssetNodeDefinition.tsx | 169 ++---------------- .../packages/core/src/assets/AssetView.tsx | 74 +++++++- .../types/AssetNodeDefinitionRunsQuery.ts | 3 +- .../asset-graph/AssetGraphExplorer.tsx | 8 +- .../src/workspace/asset-graph/AssetNode.tsx | 19 ++ .../asset-graph/InProgressRunsBanner.tsx | 37 ++++ .../asset-graph/SidebarAssetInfo.tsx | 28 +-- .../core/src/workspace/asset-graph/Utils.tsx | 14 +- .../types/InProgressRunsFragment.ts | 20 +++ 12 files changed, 215 insertions(+), 221 deletions(-) create mode 100644 js_modules/dagit/packages/core/src/workspace/asset-graph/InProgressRunsBanner.tsx create mode 100644 js_modules/dagit/packages/core/src/workspace/asset-graph/types/InProgressRunsFragment.ts diff --git a/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx b/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx index dd00fda4157d0..46ceaa447697d 100644 --- a/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx +++ b/js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx @@ -150,22 +150,24 @@ const AssetMaterializationRow: React.FC<{ )} {metadataEntries.length || hasLineage ? ( - {(metadataEntries || []).map((entry) => ( -
{entry.label} - -
Parent Materializations - -
{entry.label} + +
Parent Materializations + +
Run + {latestRun ? ( +
+ + {'Run '} + + {titleForRun({runId: latestEvent.runId})} + + + + + + + + + {latestEvent.stepKey} + + +
+ ) : ( + 'No materialization events' + )} +
RunLatest partition{latest ? latest.partition : 'No materialization events'}
Timestamp + {latestEvent ? ( + + ) : ( + 'No materialization events' + )} +
Parent assets - {latestRun ? ( -
- - {'Run '} - - {titleForRun({runId: latestEvent.runId})} - - - - - - - - - {latestEvent.stepKey} - - -
- ) : ( - 'No materialization events' - )} +
Latest partition{latest ? latest.partition : 'No materialization events'}
Timestamp
{entry.label} - {latestEvent ? ( - - ) : ( - 'No materialization events' - )} +
Parent assets - -
{entry.label} - -
{label}