[dagit] Auto-refreshing materialization + in progress run overlays on Asset graph

js_modules/dagit/packages/core/src/assets/AssetEntryRoot.tsx

@@ -1,96 +1,42 @@
 import {gql, useQuery} from '@apollo/client';
-import {BreadcrumbProps, Breadcrumbs} from '@blueprintjs/core';
 import * as React from 'react';
-import {Link, RouteComponentProps} from 'react-router-dom';
-import styled from 'styled-components/macro';
+import {RouteComponentProps} from 'react-router-dom';
 
-import {Box} from '../ui/Box';
-import {ColorsWIP} from '../ui/Colors';
 import {Loading} from '../ui/Loading';
 import {Page} from '../ui/Page';
-import {PageHeader} from '../ui/PageHeader';
-import {TagWIP} from '../ui/TagWIP';
-import {Heading} from '../ui/Text';
 
+import {AssetPageHeader} from './AssetPageHeader';
 import {AssetView} from './AssetView';
 import {AssetsCatalogTable} from './AssetsCatalogTable';
 import {AssetEntryRootQuery} from './types/AssetEntryRootQuery';
-import {useAssetView} from './useAssetView';
 
 export const AssetEntryRoot: React.FC<RouteComponentProps> = ({match}) => {
   const currentPath: string[] = (match.params['0'] || '')
     .split('/')
     .filter((x: string) => x)
     .map(decodeURIComponent);
 
-  const [view] = useAssetView();
-
   const queryResult = useQuery<AssetEntryRootQuery>(ASSET_ENTRY_ROOT_QUERY, {
     variables: {assetKey: {path: currentPath}},
   });
 
-  const breadcrumbs = React.useMemo(() => {
-    if (currentPath.length === 1 || view !== 'directory') {
-      return null;
-    }
-
-    const list: BreadcrumbProps[] = [];
-    currentPath.reduce((accum: string, elem: string) => {
-      const href = `${accum}/${encodeURIComponent(elem)}`;
-      list.push({text: elem, href});
-      return href;
-    }, '/instance/assets');
-
-    return list;
-  }, [currentPath, view]);
-
-  return (
+  return queryResult.loading ? (
     <Page>
-      <PageHeader
-        title={
-          view !== 'directory' || !breadcrumbs ? (
-            <Heading>{currentPath[currentPath.length - 1]}</Heading>
-          ) : (
-            <Box
-              flex={{alignItems: 'center', gap: 4}}
-              style={{maxWidth: '600px', overflow: 'hidden'}}
-            >
-              <Breadcrumbs
-                items={breadcrumbs}
-                breadcrumbRenderer={({text, href}) => (
-                  <Heading>
-                    <BreadcrumbLink to={href || '#'}>{text}</BreadcrumbLink>
-                  </Heading>
-                )}
-                currentBreadcrumbRenderer={({text}) => <Heading>{text}</Heading>}
-              />
-            </Box>
-          )
-        }
-        tags={<TagWIP icon="asset">Asset</TagWIP>}
-      />
-      <Loading queryResult={queryResult}>
-        {({assetOrError}) => {
-          if (assetOrError.__typename === 'AssetNotFoundError') {
-            return <AssetsCatalogTable prefixPath={currentPath} />;
-          }
-
-          return <AssetView assetKey={{path: currentPath}} />;
-        }}
-      </Loading>
+      <AssetPageHeader currentPath={currentPath} />
+      <Loading queryResult={queryResult}>{() => null}</Loading>
+    </Page>
+  ) : queryResult.data?.assetOrError.__typename === 'AssetNotFoundError' ? (
+    <Page>
+      <AssetPageHeader currentPath={currentPath} />
+      <AssetsCatalogTable prefixPath={currentPath} />
+    </Page>
+  ) : (
+    <Page>
+      <AssetView assetKey={{path: currentPath}} />
     </Page>
   );
 };
 
-const BreadcrumbLink = styled(Link)`
-  color: ${ColorsWIP.Gray800};
-
-  :hover,
-  :active {
-    color: ${ColorsWIP.Gray800};
-  }
-`;
-
 const ASSET_ENTRY_ROOT_QUERY = gql`
   query AssetEntryRootQuery($assetKey: AssetKeyInput!) {
     assetOrError(assetKey: $assetKey) {

js_modules/dagit/packages/core/src/assets/AssetMaterializationTable.tsx

@@ -150,22 +150,24 @@ const AssetMaterializationRow: React.FC<{
             )}
             {metadataEntries.length || hasLineage ? (
               <DetailsTable>
-                {(metadataEntries || []).map((entry) => (
-                  <tr key={`metadata-${entry.label}`}>
-                    <td>{entry.label}</td>
-                    <td>
-                      <MetadataEntry entry={entry} expandSmallValues={true} />
-                    </td>
-                  </tr>
-                ))}
-                {hasLineage && (
-                  <tr>
-                    <td>Parent Materializations</td>
-                    <td>
-                      <AssetLineageElements elements={assetLineage} timestamp={timestamp} />
-                    </td>
-                  </tr>
-                )}
+                <tbody>
+                  {(metadataEntries || []).map((entry) => (
+                    <tr key={`metadata-${entry.label}`}>
+                      <td>{entry.label}</td>
+                      <td>
+                        <MetadataEntry entry={entry} expandSmallValues={true} />
+                      </td>
+                    </tr>
+                  ))}
+                  {hasLineage && (
+                    <tr>
+                      <td>Parent Materializations</td>
+                      <td>
+                        <AssetLineageElements elements={assetLineage} timestamp={timestamp} />
+                      </td>
+                    </tr>
+                  )}
+                </tbody>
               </DetailsTable>
             ) : (
               <Box padding={{horizontal: 24, vertical: 12}}>No materialization event metadata</Box>

js_modules/dagit/packages/core/src/assets/AssetMaterializations.tsx

@@ -2,19 +2,25 @@ import {gql, useQuery} from '@apollo/client';
 import flatMap from 'lodash/flatMap';
 import uniq from 'lodash/uniq';
 import * as React from 'react';
+import {Link} from 'react-router-dom';
+import styled from 'styled-components/macro';
 
+import {SidebarSection} from '../pipelines/SidebarComponents';
 import {METADATA_ENTRY_FRAGMENT} from '../runs/MetadataEntry';
 import {Box} from '../ui/Box';
 import {ButtonGroup} from '../ui/ButtonGroup';
 import {ColorsWIP} from '../ui/Colors';
+import {IconWIP} from '../ui/Icon';
 import {NonIdealState} from '../ui/NonIdealState';
 import {Spinner} from '../ui/Spinner';
-import {Subheading} from '../ui/Text';
+import {Caption, Subheading} from '../ui/Text';
+import {InProgressRunsBanner} from '../workspace/asset-graph/InProgressRunsBanner';
 
 import {ASSET_LINEAGE_FRAGMENT} from './AssetLineageElements';
 import {AssetMaterializationTable} from './AssetMaterializationTable';
 import {AssetValueGraph} from './AssetValueGraph';
 import {AssetViewParams} from './AssetView';
+import {LatestMaterializationMetadata} from './LastMaterializationMetadata';
 import {AssetKey, AssetNumericHistoricalData} from './types';
 import {AssetMaterializationFragment} from './types/AssetMaterializationFragment';
 import {
@@ -26,29 +32,45 @@ import {HistoricalMaterialization, useMaterializationBuckets} from './useMateria
 interface Props {
   assetKey: AssetKey;
   asSidebarSection?: boolean;
+  inProgressRunIds?: string[];
   params: AssetViewParams;
   paramsTimeWindowOnly: boolean;
   setParams: (params: AssetViewParams) => void;
+
+  // This timestamp is a "hint", when it changes this component will refetch
+  // to retrieve new data. Just don't want to poll the entire table query.
+  assetLastMaterializedAt: string | undefined;
 }
+
 const LABEL_STEP_EXECUTION_TIME = 'Step Execution Time';
 
 export const AssetMaterializations: React.FC<Props> = ({
   assetKey,
+  assetLastMaterializedAt,
   asSidebarSection,
   params,
-  setParams,
   paramsTimeWindowOnly,
+  setParams,
+  inProgressRunIds,
 }) => {
-  const {data, loading} = useQuery<AssetMaterializationsQuery, AssetMaterializationsQueryVariables>(
-    ASSET_MATERIALIZATIONS_QUERY,
-    {
-      variables: {
-        assetKey: {path: assetKey.path},
-        before: paramsTimeWindowOnly && params.asOf ? `${Number(params.asOf) + 1}` : undefined,
-        limit: 200,
-      },
+  const {data, loading, refetch} = useQuery<
+    AssetMaterializationsQuery,
+    AssetMaterializationsQueryVariables
+  >(ASSET_MATERIALIZATIONS_QUERY, {
+    variables: {
+      assetKey: {path: assetKey.path},
+      before: paramsTimeWindowOnly && params.asOf ? `${Number(params.asOf) + 1}` : undefined,
+      limit: 200,
     },
-  );
+  });
+
+  React.useEffect(() => {
+    if (paramsTimeWindowOnly) {
+      return;
+    }
+    console.log('refetch');
+    refetch();
+  }, [paramsTimeWindowOnly, assetLastMaterializedAt, refetch]);
 
   const asset = data?.assetOrError.__typename === 'Asset' ? data?.assetOrError : null;
   const materializations = asset?.assetMaterializations || [];
@@ -77,6 +99,51 @@ export const AssetMaterializations: React.FC<Props> = ({
     );
   }
 
+  const notYetMaterializedRunIds = (inProgressRunIds || []).filter(
+    (runId) =>
+      !materializations.some(
+        (m) => m.runOrError.__typename === 'Run' && m.runOrError.runId === runId,
+      ),
+  );
+
+  if (asSidebarSection) {
+    const latest = materializations[0];
+    return (
+      <>
+        <InProgressRunsBanner runIds={inProgressRunIds || []} />
+        <SidebarSection title={'Materialization in Last Run'}>
+          <>
+            {latest ? (
+              <div style={{margin: -1, maxWidth: '100%', overflowX: 'auto'}}>
+                <LatestMaterializationMetadata latest={latest} />
+              </div>
+            ) : (
+              <Box
+                margin={{horizontal: 24, bottom: 24, top: 12}}
+                style={{color: ColorsWIP.Gray500, fontSize: '0.8rem'}}
+              >
+                No materializations found
+              </Box>
+            )}
+            <Box margin={{bottom: 12, horizontal: 12, top: 20}}>
+              <AssetCatalogLink to={`/instance/assets/${assetKey.path.join('/')}`}>
+                {'View All in Asset Catalog '}
+                <IconWIP name="open_in_new" color={ColorsWIP.Blue500} />
+              </AssetCatalogLink>
+            </Box>
+          </>
+        </SidebarSection>
+        <SidebarSection title={'Materialization Plots'}>
+          <AssetMaterializationGraphs
+            xAxis={xAxis}
+            asSidebarSection
+            assetMaterializations={reversed}
+          />
+        </SidebarSection>
+      </>
+    );
+  }
+
   if (!reversed.length) {
     return (
       <Box padding={{vertical: 20}}>
@@ -89,19 +156,13 @@ export const AssetMaterializations: React.FC<Props> = ({
     );
   }
 
-  if (asSidebarSection) {
-    return (
-      <AssetMaterializationGraphs xAxis={xAxis} asSidebarSection assetMaterializations={reversed} />
-    );
-  }
-
   return (
     <Box style={{display: 'flex'}}>
       <Box style={{flex: 1}}>
         <Box
           flex={{justifyContent: 'space-between', alignItems: 'center'}}
           padding={{vertical: 16, horizontal: 24}}
-          border={{side: 'top', width: 1, color: ColorsWIP.KeylineGray}}
+          style={{marginBottom: -1}}
         >
           <Subheading>Materializations</Subheading>
           {hasPartitions ? (
@@ -117,6 +178,7 @@ export const AssetMaterializations: React.FC<Props> = ({
             </div>
           ) : null}
         </Box>
+        <InProgressRunsBanner runIds={notYetMaterializedRunIds} />
         <AssetMaterializationTable
           hasPartitions={hasPartitions}
           hasLineage={hasLineage}
@@ -166,14 +228,33 @@ const AssetMaterializationGraphs: React.FC<{
         }}
       >
         {[...graphedLabels].sort().map((label) => (
-          <AssetValueGraph
+          <Box
             key={label}
-            label={label}
-            width={'100%'}
-            data={graphDataByMetadataLabel[label]}
-            xHover={xHover}
-            onHoverX={(x) => x !== xHover && setXHover(x)}
-          />
+            style={{width: '100%'}}
+            border={{side: 'bottom', width: 1, color: ColorsWIP.KeylineGray}}
+          >
+            {props.asSidebarSection ? (
+              <Box padding={{horizontal: 24, top: 8}}>
+                <Caption style={{fontWeight: 700}}>{label}</Caption>
+              </Box>
+            ) : (
+              <Box
+                padding={{horizontal: 24, vertical: 16}}
+                border={{side: 'bottom', width: 1, color: ColorsWIP.KeylineGray}}
+              >
+                <Subheading>{label}</Subheading>
+              </Box>
+            )}
+            <Box padding={{horizontal: 24, vertical: 16}}>
+              <AssetValueGraph
+                label={label}
+                width={'100%'}
+                data={graphDataByMetadataLabel[label]}
+                xHover={xHover}
+                onHoverX={(x) => x !== xHover && setXHover(x)}
+              />
+            </Box>
+          </Box>
         ))}
       </div>
       {xAxis === 'partition' && (
@@ -294,6 +375,7 @@ const ASSET_MATERIALIZATIONS_QUERY = gql`
         key {
           path
         }
+
         assetMaterializations(limit: $limit, beforeTimestampMillis: $before) {
           ...AssetMaterializationFragment
         }
@@ -340,3 +422,11 @@ const ASSET_MATERIALIZATIONS_QUERY = gql`
   ${METADATA_ENTRY_FRAGMENT}
   ${ASSET_LINEAGE_FRAGMENT}
 `;
+
+const AssetCatalogLink = styled(Link)`
+  display: flex;
+  gap: 5px;
+  align-items: center;
+  justify-content: flex-end;
+  margin-top: -10px;
+`;