From 55a94e7e367ffcb409303262e346efec96d39721 Mon Sep 17 00:00:00 2001 From: Sam Black Date: Tue, 30 Jul 2024 15:21:16 -0400 Subject: [PATCH 01/18] UI for Documentation Propagation --- .../components/SchemaDescriptionField.tsx | 47 +- .../components/legacy/DescriptionModal.tsx | 31 +- .../shared/propagation/PropagationDetails.tsx | 71 + .../propagation/PropagationEntityLink.tsx | 50 + .../app/entity/shared/propagation/utils.ts | 24 + .../SchemaFieldDrawer/FieldDescription.tsx | 40 +- .../utils/getFieldDescriptionDetails.ts | 25 + .../Schema/utils/useDescriptionRenderer.tsx | 17 +- .../src/app/entity/shared/useGetEntities.ts | 18 + smoke-test/tests/cypress/cypress.config.js | 42 +- .../cypress/e2e/actions/docPropagation.js | 24 + smoke-test/tests/cypress/data.json | 4370 +++++++++-------- 12 files changed, 2557 insertions(+), 2202 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx create mode 100644 datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx create mode 100644 datahub-web-react/src/app/entity/shared/propagation/utils.ts create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts create mode 100644 datahub-web-react/src/app/entity/shared/useGetEntities.ts create mode 100644 smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx index ce8d03fbdc960..e30cd1405ef9b 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx @@ -5,6 +5,8 @@ import styled from 'styled-components'; import { FetchResult } from '@apollo/client'; import { UpdateDatasetMutation } from '../../../../../../graphql/dataset.generated'; +import { StringMapEntry } from '../../../../../../types.generated'; +import PropagationDetails from '../../../../shared/propagation/PropagationDetails'; import UpdateDescriptionModal from '../../../../shared/components/legacy/DescriptionModal'; import StripMarkdownText, { removeMarkdown } from '../../../../shared/components/styled/StripMarkdownText'; import SchemaEditableContext from '../../../../../shared/SchemaEditableContext'; @@ -28,6 +30,11 @@ const ExpandedActions = styled.div` height: 10px; `; +const DescriptionWrapper = styled.span` + display: inline-flex; + align-items: center; +`; + const DescriptionContainer = styled.div` position: relative; display: flex; @@ -105,6 +112,8 @@ type Props = { isEdited?: boolean; isReadOnly?: boolean; businessAttributeDescription?: string; + isPropagated?: boolean; + sourceDetail?: StringMapEntry[] | null; }; const ABBREVIATED_LIMIT = 80; @@ -120,6 +129,8 @@ export default function DescriptionField({ original, isReadOnly, businessAttributeDescription, + isPropagated, + sourceDetail, }: Props) { const [showAddModal, setShowAddModal] = useState(false); const overLimit = removeMarkdown(description).length > 80; @@ -163,7 +174,7 @@ export default function DescriptionField({ return ( - {expanded || !overLimit ? ( + {expanded ? ( <> {!!description && } {!!description && (EditButton || overLimit) && ( @@ -184,25 +195,29 @@ export default function DescriptionField({ ) : ( <> - - { - e.stopPropagation(); - handleExpanded(true); - }} - > - Read More - - - } + // readMore={ + // <> + // { + // e.stopPropagation(); + // handleExpanded(true); + // }} + // > + // Read More + // + // + // } suffix={EditButton} shouldWrap - > + > */} + + {isPropagated && } +   {description} - + + {/* */} )} {isEdited && (edited)} diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx index 0e899bc391e0a..8d14ab6ded312 100644 --- a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx @@ -19,16 +19,29 @@ const StyledViewer = styled(Editor)` } `; +const OriginalDocumentation = styled(Form.Item)` + margin-bottom: 0; +`; + type Props = { title: string; description?: string | undefined; original?: string | undefined; + propagatedDescription?: string | undefined; onClose: () => void; onSubmit: (description: string) => void; isAddDesc?: boolean; }; -export default function UpdateDescriptionModal({ title, description, original, onClose, onSubmit, isAddDesc }: Props) { +export default function UpdateDescriptionModal({ + title, + description, + original, + propagatedDescription, + onClose, + onSubmit, + isAddDesc, +}: Props) { const [updatedDesc, setDesc] = useState(description || original || ''); const handleEditorKeyDown = (event: React.KeyboardEvent) => { @@ -64,17 +77,17 @@ export default function UpdateDescriptionModal({ title, description, original, o >
- + {!isAddDesc && description && original && ( - Original:}> + Original:}> - + + )} + {!isAddDesc && description && propagatedDescription && ( + Propagated:}> + + )}
diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx new file mode 100644 index 0000000000000..1a2f9de10ce72 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -0,0 +1,71 @@ +import React from 'react'; +import styled from 'styled-components'; +import { ThunderboltOutlined } from '@ant-design/icons'; +import { Popover } from 'antd'; +import { StringMapEntry } from '../../../../types.generated'; +import PropagationEntityLink from './PropagationEntityLink'; +import { usePropagationDetails } from './utils'; + +const PopoverWrapper = styled.div` + display: flex; + flex-direction: column; + gap: 4px; +`; + +const PropagateThunderbolt = styled(ThunderboltOutlined)` + color: #8088a3; + font-weight: bold; + &:hover { + color: #4b39bc; + } +`; + +const EntityWrapper = styled.span` + display: inline-flex; + align-items: center; + gap: 2px; +`; + +const PropagationPrefix = styled.span` + color: black; +`; + +interface Props { + sourceDetail?: StringMapEntry[] | null; +} + +export default function PropagationDetails({ sourceDetail }: Props) { + const { + isPropagated, + origin: { entity: originEntity }, + via: { entity: viaEntity }, + } = usePropagationDetails(sourceDetail); + + if (!sourceDetail || !isPropagated) return null; + + const popoverContent = + originEntity || viaEntity ? ( + + Propagated description +
+ {viaEntity && ( + + via: + + + )} + {originEntity && originEntity.urn !== viaEntity?.urn && ( + + origin: + + + )} + + ) : undefined; + + return ( + + + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx new file mode 100644 index 0000000000000..b41fb51918837 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Link } from 'react-router-dom'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { Entity, EntityType, SchemaFieldEntity } from '../../../../types.generated'; +import { GenericEntityProperties } from '../types'; + +const PreviewImage = styled.img<{ size: number }>` + height: ${(props) => props.size}px; + width: ${(props) => props.size}px; + min-width: ${(props) => props.size}px; + object-fit: contain; + background-color: transparent; + margin: 0 3px; +`; + +const StyledLink = styled(Link)` + display: flex; + align-items: center; +`; + +interface Props { + entity: Entity; +} + +export default function PropagationEntityLink({ entity }: Props) { + const entityRegistry = useEntityRegistry(); + + const isSchemaField = entity.type === EntityType.SchemaField; + const baseEntity = isSchemaField ? (entity as SchemaFieldEntity).parent : entity; + + const logoUrl = (baseEntity as GenericEntityProperties)?.platform?.properties?.logoUrl || ''; + let entityUrl = entityRegistry.getEntityUrl(baseEntity.type, baseEntity.urn); + let entityDisplayName = entityRegistry.getDisplayName(baseEntity.type, baseEntity); + + if (isSchemaField) { + entityUrl = `${entityUrl}/${encodeURIComponent('Columns')}?schemaFilter=${encodeURIComponent( + (entity as SchemaFieldEntity).fieldPath, + )}`; + const schemaFieldName = entityRegistry.getDisplayName(entity.type, entity); + entityDisplayName = `${entityDisplayName}.${schemaFieldName}`; + } + + return ( + + + {entityDisplayName} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/propagation/utils.ts b/datahub-web-react/src/app/entity/shared/propagation/utils.ts new file mode 100644 index 0000000000000..d8b4d4d931f4e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/utils.ts @@ -0,0 +1,24 @@ +import { StringMapEntry } from '../../../../types.generated'; +import { useGetEntities } from '../useGetEntities'; + +export function usePropagationDetails(sourceDetail?: StringMapEntry[] | null) { + const isPropagated = !!sourceDetail?.find((mapEntry) => mapEntry.key === 'propagated' && mapEntry.value === 'true'); + const originEntityUrn = sourceDetail?.find((mapEntry) => mapEntry.key === 'origin')?.value || ''; + const viaEntityUrn = sourceDetail?.find((mapEntry) => mapEntry.key === 'via')?.value || ''; + + const entities = useGetEntities([originEntityUrn, viaEntityUrn]); + const originEntity = entities.find((e) => e.urn === originEntityUrn); + const viaEntity = entities.find((e) => e.urn === viaEntityUrn); + + return { + isPropagated, + origin: { + urn: originEntityUrn, + entity: originEntity, + }, + via: { + urn: viaEntityUrn, + entity: viaEntity, + }, + }; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx index be95cba3ab4f0..e64a1436b0b1c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx @@ -6,6 +6,8 @@ import styled from 'styled-components'; import { SectionHeader, StyledDivider } from './components'; import UpdateDescriptionModal from '../../../../../components/legacy/DescriptionModal'; import { EditableSchemaFieldInfo, SchemaField, SubResourceType } from '../../../../../../../../types.generated'; +import { getFieldDescriptionDetails } from '../../utils/getFieldDescriptionDetails'; +import PropagationDetails from '../../../../../propagation/PropagationDetails'; import DescriptionSection from '../../../../../containers/profile/sidebar/AboutSection/DescriptionSection'; import { useEntityData, useMutationUrn, useRefetch } from '../../../../../EntityContext'; import { useSchemaRefetch } from '../../SchemaContext'; @@ -13,11 +15,6 @@ import { useUpdateDescriptionMutation } from '../../../../../../../../graphql/mu import analytics, { EntityActionType, EventType } from '../../../../../../../analytics'; import SchemaEditableContext from '../../../../../../../shared/SchemaEditableContext'; -const DescriptionWrapper = styled.div` - display: flex; - justify-content: space-between; -`; - const EditIcon = styled(Button)` border: none; box-shadow: none; @@ -25,6 +22,13 @@ const EditIcon = styled(Button)` width: 20px; `; +const DescriptionWrapper = styled.div` + display: flex; + gap: 4px; + align-items: center; + justify-content: space-between; +`; + interface Props { expandedField: SchemaField; editableFieldInfo?: EditableSchemaFieldInfo; @@ -76,7 +80,13 @@ export default function FieldDescription({ expandedField, editableFieldInfo }: P }, }); - const displayedDescription = editableFieldInfo?.description || expandedField.description; + const { schemaFieldEntity, description } = expandedField; + const { displayedDescription, isPropagated, sourceDetail, propagatedDescription } = getFieldDescriptionDetails({ + schemaFieldEntity, + editableFieldInfo, + defaultDescription: description, + }); + const baDescription = expandedField?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.properties ?.description; @@ -87,12 +97,17 @@ export default function FieldDescription({ expandedField, editableFieldInfo }: P
Description - + + {isPropagated && } + {!!displayedDescription && ( + + )} +
{isSchemaEditable && ( )} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts new file mode 100644 index 0000000000000..6434baddb77a6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts @@ -0,0 +1,25 @@ +import { EditableSchemaFieldInfo, SchemaFieldEntity } from '../../../../../../../types.generated'; + +interface Props { + schemaFieldEntity?: SchemaFieldEntity | null; + editableFieldInfo?: EditableSchemaFieldInfo; + defaultDescription?: string | null; +} + +export function getFieldDescriptionDetails({ schemaFieldEntity, editableFieldInfo, defaultDescription }: Props) { + const documentation = schemaFieldEntity?.documentation?.documentations?.[0]; + const isUsingDocumentationAspect = !editableFieldInfo?.description && !!documentation; + const isPropagated = + isUsingDocumentationAspect && + !!documentation?.attribution?.sourceDetail?.find( + (mapEntry) => mapEntry.key === 'propagated' && mapEntry.value === 'true', + ); + + const displayedDescription = + editableFieldInfo?.description || documentation?.documentation || defaultDescription || ''; + + const sourceDetail = documentation?.attribution?.sourceDetail; + const propagatedDescription = documentation?.documentation; + + return { displayedDescription, isPropagated, sourceDetail, propagatedDescription }; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx index 73e6d2ca6e9b3..bb70c2cb49303 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx @@ -6,6 +6,7 @@ import { useUpdateDescriptionMutation } from '../../../../../../../graphql/mutat import { useMutationUrn, useRefetch } from '../../../../EntityContext'; import { useSchemaRefetch } from '../SchemaContext'; import { pathMatchesNewPath } from '../../../../../dataset/profile/schema/utils/utils'; +import { getFieldDescriptionDetails } from './getFieldDescriptionDetails'; export default function useDescriptionRenderer(editableSchemaMetadata: EditableSchemaMetadata | null | undefined) { const urn = useMutationUrn(); @@ -21,10 +22,16 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS }; return (description: string, record: SchemaField, index: number): JSX.Element => { - const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find( - (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), + const editableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find((candidateEditableFieldInfo) => + pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), ); - const displayedDescription = relevantEditableFieldInfo?.description || description; + const { schemaFieldEntity } = record; + const { displayedDescription, isPropagated, sourceDetail } = getFieldDescriptionDetails({ + schemaFieldEntity, + editableFieldInfo, + defaultDescription: description, + }); + const sanitizedDescription = DOMPurify.sanitize(displayedDescription); const original = record.description ? DOMPurify.sanitize(record.description) : undefined; const businessAttributeDescription = @@ -43,7 +50,7 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS baExpanded={!!expandedBARows[index]} description={sanitizedDescription} original={original} - isEdited={!!relevantEditableFieldInfo?.description} + isEdited={!!editableFieldInfo?.description} onUpdate={(updatedDescription) => updateDescription({ variables: { @@ -56,6 +63,8 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS }, }).then(refresh) } + isPropagated={isPropagated} + sourceDetail={sourceDetail} isReadOnly /> ); diff --git a/datahub-web-react/src/app/entity/shared/useGetEntities.ts b/datahub-web-react/src/app/entity/shared/useGetEntities.ts new file mode 100644 index 0000000000000..9391bc17d7a8a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/useGetEntities.ts @@ -0,0 +1,18 @@ +import { useEffect, useState } from 'react'; +import { useGetEntitiesQuery } from '../../../graphql/entity.generated'; +import { Entity } from '../../../types.generated'; + +export function useGetEntities(urns: string[]): Entity[] { + const [verifiedUrns, setVerifiedUrns] = useState([]); + + useEffect(() => { + urns.forEach((urn) => { + if (urn.startsWith('urn:li:') && !verifiedUrns.includes(urn)) { + setVerifiedUrns((prevUrns) => [...prevUrns, urn]); + } + }); + }, [urns, verifiedUrns]); + + const { data } = useGetEntitiesQuery({ variables: { urns: verifiedUrns }, skip: !verifiedUrns.length }); + return (data?.entities || []) as Entity[]; +} diff --git a/smoke-test/tests/cypress/cypress.config.js b/smoke-test/tests/cypress/cypress.config.js index 7c3863ad869e3..cfc515766097d 100644 --- a/smoke-test/tests/cypress/cypress.config.js +++ b/smoke-test/tests/cypress/cypress.config.js @@ -2,25 +2,25 @@ const { defineConfig } = require("cypress"); module.exports = defineConfig({ - chromeWebSecurity: false, - viewportHeight: 960, - viewportWidth: 1536, - projectId: "hkrxk5", - defaultCommandTimeout: 10000, - retries: { - runMode: 2, - openMode: 0, - }, - video: false, - e2e: { - // We've imported your old cypress plugins here. - // You may want to clean this up later by importing these. - setupNodeEvents(on, config) { - // eslint-disable-next-line global-require - return require("./cypress/plugins/index")(on, config); - }, - baseUrl: "http://localhost:9002/", - specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", - experimentalStudio: true, - }, + chromeWebSecurity: false, + viewportHeight: 960, + viewportWidth: 1536, + projectId: "hkrxk5", + defaultCommandTimeout: 10000, + retries: { + runMode: 2, + openMode: 0, + }, + video: false, + e2e: { + // We've imported your old cypress plugins here. + // You may want to clean this up later by importing these. + setupNodeEvents(on, config) { + // eslint-disable-next-line global-require + return require("./cypress/plugins/index")(on, config); + }, + baseUrl: "http://localhost:9002/", + specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", + experimentalStudio: true, + } }); diff --git a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js new file mode 100644 index 0000000000000..49132bf53327c --- /dev/null +++ b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js @@ -0,0 +1,24 @@ +const testId = '[data-testid="docPropagationIndicator"]'; + +describe('docPropagation', () => { + it('logs in and navigates to the schema page and checks for docPropagationIndicator', () => { + cy.login(); + cy.visit('http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter='); + + // verify that the indicator exists in the table + cy.get(testId).should('exist'); + + // click on the table row + cy.get('[data-row-key="user_id"]').click(); + + // verify that the indicator exists in id="entity-profile-sidebar" + cy.get('[id="entity-profile-sidebar"]') + .then(($sidebar) => { + if ($sidebar.find(testId).length) return testId; + return null; + }) + .then((selector) => { + cy.get(selector).should('exist'); + }); + }) +}); \ No newline at end of file diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index 5253b7a33b085..b2df28934ce1c 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -1,2141 +1,2231 @@ [ - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/prod/kafka/SampleKafkaDataset"] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": null, - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleKafkaSchema", - "platform": "urn:li:dataPlatform:kafka", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=boolean].field_foo_2", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "globalTags": { - "tags": [{ "tag": "urn:li:tag:NeedsDocumentation" }] - }, - "recursive": false - }, - { - "fieldPath": "[version=2.0].[type=boolean].field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "[version=2.0].[key=True].[type=int].id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id specifying which partition the message should go to" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hdfs dataset", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/prod/hdfs/SampleCypressHdfsDataset"] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "editableSchemaFieldInfo": [ - { - "fieldPath": "shipment_info", - "globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] }, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHdfsSchema", - "platform": "urn:li:dataPlatform:hdfs", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "shipment_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.date", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info date description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "Date", - "recursive": false - }, - { - "fieldPath": "shipment_info.target", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info target description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false - }, - { - "fieldPath": "shipment_info.destination", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info destination description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lat", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lat" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lng", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lng" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { "tag": "urn:li:tag:Cypress" }, - { "tag": "urn:li:tag:Cypress2" } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hive dataset", - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false, - "isPartOfKey": true - }, - { - "fieldPath": "field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table where each row represents a single log event", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "event_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of your logging event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "event_data", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Data of your event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "TS the event was ingested" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "browser", - "jsonPath": null, - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day. Creted from log events.", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users deleted on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who was deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Timestamp user was deleted at" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false - }, - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "browser_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the browser" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "session_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the session" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "deletion_reason", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Why the user chose to deactivate" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - } - ], - "primaryKeys": ["user_name"], - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user session", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Creations", - "description": "Constructs the fct_users_created from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Deletions", - "description": "Constructs the fct_users_deleted from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "name": "Users", - "description": "Constructs the fct_users_deleted and fct_users_created tables", - "project": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 1", - "description": "Baz Chart 1", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 2", - "description": "Baz Chart 2", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": { - "array": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - } - ] - }, - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(looker,cypress_baz)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpGroup:bfoo", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "title": "Baz Dashboard", - "description": "Baz Dashboard", - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - }, - "charts": [ - "urn:li:chart:(looker,cypress_baz1)", - "urn:li:chart:(looker,cypress_baz2)" - ], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { - "urn": "urn:li:glossaryNode:CypressNode", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { - "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress", - "description": "Indicates the entity is for cypress integration test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress2", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress 2", - "description": "Indicates the entity is #2 for cypress test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:CypressFeatureTag" }] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "this is a description from source system", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:CypressPrimaryKeyTag" }] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/sagemaker/cypress-feature-table"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", - "creation_time": "2021-06-24 09:48:37.035000", - "status": "Created" - }, - "description": "Yet another test feature group", - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "customProperties": { - "EnableNetworkIsolation": "True" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", - "description": "ml model description", - "date": 1420070400000, - "version": null, - "type": null, - "hyperParameters": null, - "hyperParams": [ - { - "name": "parameter-1", - "description": null, - "value": "some-value", - "createdAt": null - } - ], - "trainingMetrics": [ - { - "name": "another-metric", - "description": null, - "value": "1.0", - "createdAt": null - } - ], - "onlineMetrics": null, - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "tags": [], - "deployments": [], - "trainingJobs": [], - "downstreamJobs": [], - "groups": [ - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/sagemaker/cypress-model-package-group/cypress-model"] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { - "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { - "customProperties": { - "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", - "ModelPackageGroupDescription": "Just a model package group.", - "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", - "ModelPackageGroupStatus": "Completed" - }, - "description": "Just a model package group.", - "createdAt": 1420070400000, - "version": null - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:some-user", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/sagemaker/cypress-model-package-group"] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a definition", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "domain", - "entityUrn": "urn:li:domain:marketing", - "changeType": "UPSERT", - "aspectName": "domainProperties", - "aspect": { - "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table for health", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionInfo", - "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "incident", - "entityUrn": "urn:li:incident:test", - "changeType": "UPSERT", - "aspectName": "incidentInfo", - "aspect": { - "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "changeType": "UPSERT", - "aspectName": "incidentsSummary", - "aspect": { - "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - } -] + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka/SampleKafkaDataset" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": null, + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleKafkaSchema", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=boolean].field_foo_2", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "recursive": false + }, + { + "fieldPath": "[version=2.0].[type=boolean].field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "[version=2.0].[key=True].[type=int].id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id specifying which partition the message should go to" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hdfs dataset", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/hdfs/SampleCypressHdfsDataset" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "editableSchemaFieldInfo": [ + { + "fieldPath": "shipment_info", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + }, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:hdfs", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "shipment_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.date", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info date description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "Date", + "recursive": false + }, + { + "fieldPath": "shipment_info.target", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info target description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false + }, + { + "fieldPath": "shipment_info.destination", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info destination description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lat", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lat" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lng", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lng" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + }, + { + "tag": "urn:li:tag:Cypress2" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hive dataset", + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "field_foo", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table where each row represents a single log event", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "event_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of your logging event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "event_data", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Data of your event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "TS the event was ingested" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "browser", + "jsonPath": null, + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day. Creted from log events.", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users deleted on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who was deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Timestamp user was deleted at" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "browser_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the browser" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "session_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the session" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "deletion_reason", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Why the user chose to deactivate" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + } + ], + "primaryKeys": [ + "user_name" + ], + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user session", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Creations", + "description": "Constructs the fct_users_created from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Deletions", + "description": "Constructs the fct_users_deleted from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "name": "Users", + "description": "Constructs the fct_users_deleted and fct_users_created tables", + "project": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 1", + "description": "Baz Chart 1", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 2", + "description": "Baz Chart 2", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": { + "array": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + } + ] + }, + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(looker,cypress_baz)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:bfoo", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "title": "Baz Dashboard", + "description": "Baz Dashboard", + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + }, + "charts": [ + "urn:li:chart:(looker,cypress_baz1)", + "urn:li:chart:(looker,cypress_baz2)" + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "dashboardUrl": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { + "urn": "urn:li:glossaryNode:CypressNode", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress", + "description": "Indicates the entity is for cypress integration test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress2", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress 2", + "description": "Indicates the entity is #2 for cypress test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "description": null, + "dataType": "TEXT", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressFeatureTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "description": "this is a description from source system", + "dataType": "ORDINAL", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressPrimaryKeyTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/cypress-feature-table" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", + "creation_time": "2021-06-24 09:48:37.035000", + "status": "Created" + }, + "description": "Yet another test feature group", + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": { + "EnableNetworkIsolation": "True" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", + "description": "ml model description", + "date": 1420070400000, + "version": null, + "type": null, + "hyperParameters": null, + "hyperParams": [ + { + "name": "parameter-1", + "description": null, + "value": "some-value", + "createdAt": null + } + ], + "trainingMetrics": [ + { + "name": "another-metric", + "description": null, + "value": "1.0", + "createdAt": null + } + ], + "onlineMetrics": null, + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "tags": [], + "deployments": [], + "trainingJobs": [], + "downstreamJobs": [], + "groups": [ + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/cypress-model-package-group/cypress-model" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { + "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { + "customProperties": { + "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", + "ModelPackageGroupDescription": "Just a model package group.", + "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", + "ModelPackageGroupStatus": "Completed" + }, + "description": "Just a model package group.", + "createdAt": 1420070400000, + "version": null + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:some-user", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/cypress-model-package-group" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a definition", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "domain", + "entityUrn": "urn:li:domain:marketing", + "changeType": "UPSERT", + "aspectName": "domainProperties", + "aspect": { + "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table for health", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "incident", + "entityUrn": "urn:li:incident:test", + "changeType": "UPSERT", + "aspectName": "incidentInfo", + "aspect": { + "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "changeType": "UPSERT", + "aspectName": "incidentsSummary", + "aspect": { + "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "schemaField", + "entityUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)", + "changeType": "UPSERT", + "aspectName": "documentation", + "aspect": { + "value": "{\"documentations\":[{\"attribution\":{\"actor\":\"urn:li:corpuser:__datahub_system\",\"source\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"sourceDetail\":{\"actor\":\"urn:li:corpuser:shirshanka@acryl.io\",\"origin\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"propagated\":\"true\"},\"time\":1721422917808},\"documentation\":\"Unique identifier of user profile.\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + } +] \ No newline at end of file From fc325d2b541333c2e9f1b026ffa1ebfbdd74a642 Mon Sep 17 00:00:00 2001 From: Sam Black Date: Tue, 30 Jul 2024 16:58:55 -0400 Subject: [PATCH 02/18] Cypress prettier format fix --- smoke-test/tests/cypress/cypress.config.js | 42 +- .../cypress/e2e/actions/docPropagation.js | 40 +- smoke-test/tests/cypress/data.json | 4448 ++++++++--------- 3 files changed, 2260 insertions(+), 2270 deletions(-) diff --git a/smoke-test/tests/cypress/cypress.config.js b/smoke-test/tests/cypress/cypress.config.js index cfc515766097d..7c3863ad869e3 100644 --- a/smoke-test/tests/cypress/cypress.config.js +++ b/smoke-test/tests/cypress/cypress.config.js @@ -2,25 +2,25 @@ const { defineConfig } = require("cypress"); module.exports = defineConfig({ - chromeWebSecurity: false, - viewportHeight: 960, - viewportWidth: 1536, - projectId: "hkrxk5", - defaultCommandTimeout: 10000, - retries: { - runMode: 2, - openMode: 0, - }, - video: false, - e2e: { - // We've imported your old cypress plugins here. - // You may want to clean this up later by importing these. - setupNodeEvents(on, config) { - // eslint-disable-next-line global-require - return require("./cypress/plugins/index")(on, config); - }, - baseUrl: "http://localhost:9002/", - specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", - experimentalStudio: true, - } + chromeWebSecurity: false, + viewportHeight: 960, + viewportWidth: 1536, + projectId: "hkrxk5", + defaultCommandTimeout: 10000, + retries: { + runMode: 2, + openMode: 0, + }, + video: false, + e2e: { + // We've imported your old cypress plugins here. + // You may want to clean this up later by importing these. + setupNodeEvents(on, config) { + // eslint-disable-next-line global-require + return require("./cypress/plugins/index")(on, config); + }, + baseUrl: "http://localhost:9002/", + specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", + experimentalStudio: true, + }, }); diff --git a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js index 49132bf53327c..9cd449b3fce7c 100644 --- a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js +++ b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js @@ -1,24 +1,26 @@ const testId = '[data-testid="docPropagationIndicator"]'; -describe('docPropagation', () => { - it('logs in and navigates to the schema page and checks for docPropagationIndicator', () => { - cy.login(); - cy.visit('http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter='); +describe("docPropagation", () => { + it("logs in and navigates to the schema page and checks for docPropagationIndicator", () => { + cy.login(); + cy.visit( + "http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", + ); - // verify that the indicator exists in the table - cy.get(testId).should('exist'); + // verify that the indicator exists in the table + cy.get(testId).should("exist"); - // click on the table row - cy.get('[data-row-key="user_id"]').click(); + // click on the table row + cy.get('[data-row-key="user_id"]').click(); - // verify that the indicator exists in id="entity-profile-sidebar" - cy.get('[id="entity-profile-sidebar"]') - .then(($sidebar) => { - if ($sidebar.find(testId).length) return testId; - return null; - }) - .then((selector) => { - cy.get(selector).should('exist'); - }); - }) -}); \ No newline at end of file + // verify that the indicator exists in id="entity-profile-sidebar" + cy.get('[id="entity-profile-sidebar"]') + .then(($sidebar) => { + if ($sidebar.find(testId).length) return testId; + return null; + }) + .then((selector) => { + cy.get(selector).should("exist"); + }); + }); +}); diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index b2df28934ce1c..ce61f7c83a038 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -1,2231 +1,2219 @@ [ - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/kafka/SampleKafkaDataset" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": null, - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleKafkaSchema", - "platform": "urn:li:dataPlatform:kafka", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=boolean].field_foo_2", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:NeedsDocumentation" - } - ] - }, - "recursive": false - }, - { - "fieldPath": "[version=2.0].[type=boolean].field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "[version=2.0].[key=True].[type=int].id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id specifying which partition the message should go to" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hdfs dataset", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/hdfs/SampleCypressHdfsDataset" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "editableSchemaFieldInfo": [ - { - "fieldPath": "shipment_info", - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - }, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHdfsSchema", - "platform": "urn:li:dataPlatform:hdfs", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "shipment_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.date", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info date description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "Date", - "recursive": false - }, - { - "fieldPath": "shipment_info.target", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info target description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false - }, - { - "fieldPath": "shipment_info.destination", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info destination description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lat", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lat" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lng", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lng" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - }, - { - "tag": "urn:li:tag:Cypress2" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hive dataset", - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false, - "isPartOfKey": true - }, - { - "fieldPath": "field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table where each row represents a single log event", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "event_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of your logging event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "event_data", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Data of your event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "TS the event was ingested" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "browser", - "jsonPath": null, - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day. Creted from log events.", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users deleted on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who was deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Timestamp user was deleted at" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false - }, - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "browser_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the browser" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "session_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the session" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "deletion_reason", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Why the user chose to deactivate" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - } - ], - "primaryKeys": [ - "user_name" - ], - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user session", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Creations", - "description": "Constructs the fct_users_created from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Deletions", - "description": "Constructs the fct_users_deleted from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "name": "Users", - "description": "Constructs the fct_users_deleted and fct_users_created tables", - "project": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 1", - "description": "Baz Chart 1", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 2", - "description": "Baz Chart 2", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": { - "array": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - } - ] - }, - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(looker,cypress_baz)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpGroup:bfoo", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "title": "Baz Dashboard", - "description": "Baz Dashboard", - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - }, - "charts": [ - "urn:li:chart:(looker,cypress_baz1)", - "urn:li:chart:(looker,cypress_baz2)" - ], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { - "urn": "urn:li:glossaryNode:CypressNode", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { - "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress", - "description": "Indicates the entity is for cypress integration test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress2", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress 2", - "description": "Indicates the entity is #2 for cypress test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:CypressFeatureTag" - } - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "this is a description from source system", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:CypressPrimaryKeyTag" - } - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-feature-table" - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", - "creation_time": "2021-06-24 09:48:37.035000", - "status": "Created" - }, - "description": "Yet another test feature group", - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "customProperties": { - "EnableNetworkIsolation": "True" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", - "description": "ml model description", - "date": 1420070400000, - "version": null, - "type": null, - "hyperParameters": null, - "hyperParams": [ - { - "name": "parameter-1", - "description": null, - "value": "some-value", - "createdAt": null - } - ], - "trainingMetrics": [ - { - "name": "another-metric", - "description": null, - "value": "1.0", - "createdAt": null - } - ], - "onlineMetrics": null, - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "tags": [], - "deployments": [], - "trainingJobs": [], - "downstreamJobs": [], - "groups": [ - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-model-package-group/cypress-model" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { - "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { - "customProperties": { - "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", - "ModelPackageGroupDescription": "Just a model package group.", - "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", - "ModelPackageGroupStatus": "Completed" - }, - "description": "Just a model package group.", - "createdAt": 1420070400000, - "version": null - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:some-user", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-model-package-group" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a definition", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "domain", - "entityUrn": "urn:li:domain:marketing", - "changeType": "UPSERT", - "aspectName": "domainProperties", - "aspect": { - "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table for health", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionInfo", - "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "incident", - "entityUrn": "urn:li:incident:test", - "changeType": "UPSERT", - "aspectName": "incidentInfo", - "aspect": { - "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "changeType": "UPSERT", - "aspectName": "incidentsSummary", - "aspect": { - "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "schemaField", - "entityUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)", - "changeType": "UPSERT", - "aspectName": "documentation", - "aspect": { - "value": "{\"documentations\":[{\"attribution\":{\"actor\":\"urn:li:corpuser:__datahub_system\",\"source\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"sourceDetail\":{\"actor\":\"urn:li:corpuser:shirshanka@acryl.io\",\"origin\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"propagated\":\"true\"},\"time\":1721422917808},\"documentation\":\"Unique identifier of user profile.\"}]}", - "contentType": "application/json" - }, - "systemMetadata": null - } -] \ No newline at end of file + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/prod/kafka/SampleKafkaDataset"] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": null, + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleKafkaSchema", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=boolean].field_foo_2", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "recursive": false + }, + { + "fieldPath": "[version=2.0].[type=boolean].field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "[version=2.0].[key=True].[type=int].id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id specifying which partition the message should go to" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hdfs dataset", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/prod/hdfs/SampleCypressHdfsDataset"] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "editableSchemaFieldInfo": [ + { + "fieldPath": "shipment_info", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + }, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:hdfs", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "shipment_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.date", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info date description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "Date", + "recursive": false + }, + { + "fieldPath": "shipment_info.target", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info target description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false + }, + { + "fieldPath": "shipment_info.destination", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info destination description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lat", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lat" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lng", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lng" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + }, + { + "tag": "urn:li:tag:Cypress2" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hive dataset", + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "field_foo", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table where each row represents a single log event", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "event_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of your logging event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "event_data", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Data of your event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "TS the event was ingested" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "browser", + "jsonPath": null, + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day. Creted from log events.", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users deleted on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who was deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Timestamp user was deleted at" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "browser_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the browser" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "session_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the session" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "deletion_reason", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Why the user chose to deactivate" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + } + ], + "primaryKeys": ["user_name"], + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user session", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Creations", + "description": "Constructs the fct_users_created from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Deletions", + "description": "Constructs the fct_users_deleted from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "name": "Users", + "description": "Constructs the fct_users_deleted and fct_users_created tables", + "project": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 1", + "description": "Baz Chart 1", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 2", + "description": "Baz Chart 2", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": { + "array": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + } + ] + }, + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(looker,cypress_baz)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:bfoo", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "title": "Baz Dashboard", + "description": "Baz Dashboard", + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + }, + "charts": [ + "urn:li:chart:(looker,cypress_baz1)", + "urn:li:chart:(looker,cypress_baz2)" + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "dashboardUrl": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { + "urn": "urn:li:glossaryNode:CypressNode", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress", + "description": "Indicates the entity is for cypress integration test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress2", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress 2", + "description": "Indicates the entity is #2 for cypress test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "description": null, + "dataType": "TEXT", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressFeatureTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "description": "this is a description from source system", + "dataType": "ORDINAL", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressPrimaryKeyTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/sagemaker/cypress-feature-table"] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", + "creation_time": "2021-06-24 09:48:37.035000", + "status": "Created" + }, + "description": "Yet another test feature group", + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": { + "EnableNetworkIsolation": "True" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", + "description": "ml model description", + "date": 1420070400000, + "version": null, + "type": null, + "hyperParameters": null, + "hyperParams": [ + { + "name": "parameter-1", + "description": null, + "value": "some-value", + "createdAt": null + } + ], + "trainingMetrics": [ + { + "name": "another-metric", + "description": null, + "value": "1.0", + "createdAt": null + } + ], + "onlineMetrics": null, + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "tags": [], + "deployments": [], + "trainingJobs": [], + "downstreamJobs": [], + "groups": [ + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/sagemaker/cypress-model-package-group/cypress-model"] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { + "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { + "customProperties": { + "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", + "ModelPackageGroupDescription": "Just a model package group.", + "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", + "ModelPackageGroupStatus": "Completed" + }, + "description": "Just a model package group.", + "createdAt": 1420070400000, + "version": null + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:some-user", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/sagemaker/cypress-model-package-group"] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a definition", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "domain", + "entityUrn": "urn:li:domain:marketing", + "changeType": "UPSERT", + "aspectName": "domainProperties", + "aspect": { + "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table for health", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "incident", + "entityUrn": "urn:li:incident:test", + "changeType": "UPSERT", + "aspectName": "incidentInfo", + "aspect": { + "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "changeType": "UPSERT", + "aspectName": "incidentsSummary", + "aspect": { + "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "schemaField", + "entityUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)", + "changeType": "UPSERT", + "aspectName": "documentation", + "aspect": { + "value": "{\"documentations\":[{\"attribution\":{\"actor\":\"urn:li:corpuser:__datahub_system\",\"source\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"sourceDetail\":{\"actor\":\"urn:li:corpuser:shirshanka@acryl.io\",\"origin\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"propagated\":\"true\"},\"time\":1721422917808},\"documentation\":\"Unique identifier of user profile.\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + } +] From 53f462be42885cac77fd53b358355cca86b54f68 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 30 Jul 2024 18:16:06 -0700 Subject: [PATCH 03/18] Adding some fluff --- .../components/SchemaDescriptionField.tsx | 38 +++++------ .../shared/propagation/PropagationDetails.tsx | 64 ++++++++++--------- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx index e30cd1405ef9b..e7d986028d4a6 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx @@ -195,29 +195,29 @@ export default function DescriptionField({ ) : ( <> - {/* - // { - // e.stopPropagation(); - // handleExpanded(true); - // }} - // > - // Read More - // - // - // } - suffix={EditButton} - shouldWrap - > */} {isPropagated && }   - {description} + + { + e.stopPropagation(); + handleExpanded(true); + }} + > + Read More + + + } + suffix={EditButton} + shouldWrap + > + {description} + - {/* */} )} {isEdited && (edited)} diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index 1a2f9de10ce72..18ea438d7fc30 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -1,33 +1,29 @@ import React from 'react'; import styled from 'styled-components'; -import { ThunderboltOutlined } from '@ant-design/icons'; import { Popover } from 'antd'; import { StringMapEntry } from '../../../../types.generated'; import PropagationEntityLink from './PropagationEntityLink'; import { usePropagationDetails } from './utils'; +import { ANTD_GRAY } from '../constants'; +import { PropagateThunderbolt, PropagateThunderboltFilled } from './PropagationIcon'; const PopoverWrapper = styled.div` display: flex; flex-direction: column; - gap: 4px; `; -const PropagateThunderbolt = styled(ThunderboltOutlined)` - color: #8088a3; +const PopoverTitle = styled.div` font-weight: bold; - &:hover { - color: #4b39bc; - } + font-size: 14px; + padding: 6px 0px; `; -const EntityWrapper = styled.span` - display: inline-flex; - align-items: center; - gap: 2px; -`; - -const PropagationPrefix = styled.span` - color: black; +const PopoverDescription = styled.div` + max-width: 340px; + font-size: 14px; + color: ${ANTD_GRAY[7]}; + display: inline; + padding: 0px 0px 4px 0px; `; interface Props { @@ -46,25 +42,33 @@ export default function PropagationDetails({ sourceDetail }: Props) { const popoverContent = originEntity || viaEntity ? ( - Propagated description -
- {viaEntity && ( - - via: - - - )} - {originEntity && originEntity.urn !== viaEntity?.urn && ( - - origin: - - - )} + + This description was automatically propagated{' '} + {originEntity && originEntity.urn !== viaEntity?.urn && ( + <> + from + + )} + {viaEntity && ( + <> + via + + )} + ) : undefined; return ( - + + + Propagated Description + + } + content={popoverContent} + > ); From ca9b85ad63881a3adffe1307a648963d167e4ade Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 30 Jul 2024 18:16:41 -0700 Subject: [PATCH 04/18] adding propagation action --- .../shared/propagation/PropagationIcon.tsx | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx new file mode 100644 index 0000000000000..01b4570c4ca0d --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx @@ -0,0 +1,22 @@ +import styled from 'styled-components'; +import { ThunderboltFilled } from '@ant-design/icons'; +import { REDESIGN_COLORS } from '../constants'; + +export const PropagateThunderbolt = styled(ThunderboltFilled)` + && { + color: #a7c7fa; + } + font-size: 16px; + &:hover { + color: ${REDESIGN_COLORS.BLUE}; + } + margin-right: 4px; +`; + +export const PropagateThunderboltFilled = styled(ThunderboltFilled)` + && { + color: ${REDESIGN_COLORS.BLUE}; + } + font-size: 16px; + margin-right: 4px; +`; From 2cc7081cf16c1562ce5e5d0f76f1c75cd8190bc3 Mon Sep 17 00:00:00 2001 From: Sam Black Date: Tue, 30 Jul 2024 15:21:16 -0400 Subject: [PATCH 05/18] UI for Documentation Propagation --- .../components/SchemaDescriptionField.tsx | 47 +- .../components/legacy/DescriptionModal.tsx | 31 +- .../shared/propagation/PropagationDetails.tsx | 71 + .../propagation/PropagationEntityLink.tsx | 50 + .../app/entity/shared/propagation/utils.ts | 24 + .../SchemaFieldDrawer/FieldDescription.tsx | 40 +- .../utils/getFieldDescriptionDetails.ts | 25 + .../Schema/utils/useDescriptionRenderer.tsx | 17 +- .../src/app/entity/shared/useGetEntities.ts | 18 + smoke-test/tests/cypress/cypress.config.js | 42 +- .../cypress/e2e/actions/docPropagation.js | 24 + smoke-test/tests/cypress/data.json | 4370 +++++++++-------- 12 files changed, 2557 insertions(+), 2202 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx create mode 100644 datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx create mode 100644 datahub-web-react/src/app/entity/shared/propagation/utils.ts create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts create mode 100644 datahub-web-react/src/app/entity/shared/useGetEntities.ts create mode 100644 smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx index ce8d03fbdc960..e30cd1405ef9b 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx @@ -5,6 +5,8 @@ import styled from 'styled-components'; import { FetchResult } from '@apollo/client'; import { UpdateDatasetMutation } from '../../../../../../graphql/dataset.generated'; +import { StringMapEntry } from '../../../../../../types.generated'; +import PropagationDetails from '../../../../shared/propagation/PropagationDetails'; import UpdateDescriptionModal from '../../../../shared/components/legacy/DescriptionModal'; import StripMarkdownText, { removeMarkdown } from '../../../../shared/components/styled/StripMarkdownText'; import SchemaEditableContext from '../../../../../shared/SchemaEditableContext'; @@ -28,6 +30,11 @@ const ExpandedActions = styled.div` height: 10px; `; +const DescriptionWrapper = styled.span` + display: inline-flex; + align-items: center; +`; + const DescriptionContainer = styled.div` position: relative; display: flex; @@ -105,6 +112,8 @@ type Props = { isEdited?: boolean; isReadOnly?: boolean; businessAttributeDescription?: string; + isPropagated?: boolean; + sourceDetail?: StringMapEntry[] | null; }; const ABBREVIATED_LIMIT = 80; @@ -120,6 +129,8 @@ export default function DescriptionField({ original, isReadOnly, businessAttributeDescription, + isPropagated, + sourceDetail, }: Props) { const [showAddModal, setShowAddModal] = useState(false); const overLimit = removeMarkdown(description).length > 80; @@ -163,7 +174,7 @@ export default function DescriptionField({ return ( - {expanded || !overLimit ? ( + {expanded ? ( <> {!!description && } {!!description && (EditButton || overLimit) && ( @@ -184,25 +195,29 @@ export default function DescriptionField({ ) : ( <> - - { - e.stopPropagation(); - handleExpanded(true); - }} - > - Read More - - - } + // readMore={ + // <> + // { + // e.stopPropagation(); + // handleExpanded(true); + // }} + // > + // Read More + // + // + // } suffix={EditButton} shouldWrap - > + > */} + + {isPropagated && } +   {description} - + + {/* */} )} {isEdited && (edited)} diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx index 0e899bc391e0a..8d14ab6ded312 100644 --- a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx @@ -19,16 +19,29 @@ const StyledViewer = styled(Editor)` } `; +const OriginalDocumentation = styled(Form.Item)` + margin-bottom: 0; +`; + type Props = { title: string; description?: string | undefined; original?: string | undefined; + propagatedDescription?: string | undefined; onClose: () => void; onSubmit: (description: string) => void; isAddDesc?: boolean; }; -export default function UpdateDescriptionModal({ title, description, original, onClose, onSubmit, isAddDesc }: Props) { +export default function UpdateDescriptionModal({ + title, + description, + original, + propagatedDescription, + onClose, + onSubmit, + isAddDesc, +}: Props) { const [updatedDesc, setDesc] = useState(description || original || ''); const handleEditorKeyDown = (event: React.KeyboardEvent) => { @@ -64,17 +77,17 @@ export default function UpdateDescriptionModal({ title, description, original, o >
- + {!isAddDesc && description && original && ( - Original:}> + Original:}> - + + )} + {!isAddDesc && description && propagatedDescription && ( + Propagated:}> + + )}
diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx new file mode 100644 index 0000000000000..1a2f9de10ce72 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -0,0 +1,71 @@ +import React from 'react'; +import styled from 'styled-components'; +import { ThunderboltOutlined } from '@ant-design/icons'; +import { Popover } from 'antd'; +import { StringMapEntry } from '../../../../types.generated'; +import PropagationEntityLink from './PropagationEntityLink'; +import { usePropagationDetails } from './utils'; + +const PopoverWrapper = styled.div` + display: flex; + flex-direction: column; + gap: 4px; +`; + +const PropagateThunderbolt = styled(ThunderboltOutlined)` + color: #8088a3; + font-weight: bold; + &:hover { + color: #4b39bc; + } +`; + +const EntityWrapper = styled.span` + display: inline-flex; + align-items: center; + gap: 2px; +`; + +const PropagationPrefix = styled.span` + color: black; +`; + +interface Props { + sourceDetail?: StringMapEntry[] | null; +} + +export default function PropagationDetails({ sourceDetail }: Props) { + const { + isPropagated, + origin: { entity: originEntity }, + via: { entity: viaEntity }, + } = usePropagationDetails(sourceDetail); + + if (!sourceDetail || !isPropagated) return null; + + const popoverContent = + originEntity || viaEntity ? ( + + Propagated description +
+ {viaEntity && ( + + via: + + + )} + {originEntity && originEntity.urn !== viaEntity?.urn && ( + + origin: + + + )} + + ) : undefined; + + return ( + + + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx new file mode 100644 index 0000000000000..b41fb51918837 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Link } from 'react-router-dom'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { Entity, EntityType, SchemaFieldEntity } from '../../../../types.generated'; +import { GenericEntityProperties } from '../types'; + +const PreviewImage = styled.img<{ size: number }>` + height: ${(props) => props.size}px; + width: ${(props) => props.size}px; + min-width: ${(props) => props.size}px; + object-fit: contain; + background-color: transparent; + margin: 0 3px; +`; + +const StyledLink = styled(Link)` + display: flex; + align-items: center; +`; + +interface Props { + entity: Entity; +} + +export default function PropagationEntityLink({ entity }: Props) { + const entityRegistry = useEntityRegistry(); + + const isSchemaField = entity.type === EntityType.SchemaField; + const baseEntity = isSchemaField ? (entity as SchemaFieldEntity).parent : entity; + + const logoUrl = (baseEntity as GenericEntityProperties)?.platform?.properties?.logoUrl || ''; + let entityUrl = entityRegistry.getEntityUrl(baseEntity.type, baseEntity.urn); + let entityDisplayName = entityRegistry.getDisplayName(baseEntity.type, baseEntity); + + if (isSchemaField) { + entityUrl = `${entityUrl}/${encodeURIComponent('Columns')}?schemaFilter=${encodeURIComponent( + (entity as SchemaFieldEntity).fieldPath, + )}`; + const schemaFieldName = entityRegistry.getDisplayName(entity.type, entity); + entityDisplayName = `${entityDisplayName}.${schemaFieldName}`; + } + + return ( + + + {entityDisplayName} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/propagation/utils.ts b/datahub-web-react/src/app/entity/shared/propagation/utils.ts new file mode 100644 index 0000000000000..d8b4d4d931f4e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/utils.ts @@ -0,0 +1,24 @@ +import { StringMapEntry } from '../../../../types.generated'; +import { useGetEntities } from '../useGetEntities'; + +export function usePropagationDetails(sourceDetail?: StringMapEntry[] | null) { + const isPropagated = !!sourceDetail?.find((mapEntry) => mapEntry.key === 'propagated' && mapEntry.value === 'true'); + const originEntityUrn = sourceDetail?.find((mapEntry) => mapEntry.key === 'origin')?.value || ''; + const viaEntityUrn = sourceDetail?.find((mapEntry) => mapEntry.key === 'via')?.value || ''; + + const entities = useGetEntities([originEntityUrn, viaEntityUrn]); + const originEntity = entities.find((e) => e.urn === originEntityUrn); + const viaEntity = entities.find((e) => e.urn === viaEntityUrn); + + return { + isPropagated, + origin: { + urn: originEntityUrn, + entity: originEntity, + }, + via: { + urn: viaEntityUrn, + entity: viaEntity, + }, + }; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx index be95cba3ab4f0..e64a1436b0b1c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx @@ -6,6 +6,8 @@ import styled from 'styled-components'; import { SectionHeader, StyledDivider } from './components'; import UpdateDescriptionModal from '../../../../../components/legacy/DescriptionModal'; import { EditableSchemaFieldInfo, SchemaField, SubResourceType } from '../../../../../../../../types.generated'; +import { getFieldDescriptionDetails } from '../../utils/getFieldDescriptionDetails'; +import PropagationDetails from '../../../../../propagation/PropagationDetails'; import DescriptionSection from '../../../../../containers/profile/sidebar/AboutSection/DescriptionSection'; import { useEntityData, useMutationUrn, useRefetch } from '../../../../../EntityContext'; import { useSchemaRefetch } from '../../SchemaContext'; @@ -13,11 +15,6 @@ import { useUpdateDescriptionMutation } from '../../../../../../../../graphql/mu import analytics, { EntityActionType, EventType } from '../../../../../../../analytics'; import SchemaEditableContext from '../../../../../../../shared/SchemaEditableContext'; -const DescriptionWrapper = styled.div` - display: flex; - justify-content: space-between; -`; - const EditIcon = styled(Button)` border: none; box-shadow: none; @@ -25,6 +22,13 @@ const EditIcon = styled(Button)` width: 20px; `; +const DescriptionWrapper = styled.div` + display: flex; + gap: 4px; + align-items: center; + justify-content: space-between; +`; + interface Props { expandedField: SchemaField; editableFieldInfo?: EditableSchemaFieldInfo; @@ -76,7 +80,13 @@ export default function FieldDescription({ expandedField, editableFieldInfo }: P }, }); - const displayedDescription = editableFieldInfo?.description || expandedField.description; + const { schemaFieldEntity, description } = expandedField; + const { displayedDescription, isPropagated, sourceDetail, propagatedDescription } = getFieldDescriptionDetails({ + schemaFieldEntity, + editableFieldInfo, + defaultDescription: description, + }); + const baDescription = expandedField?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.properties ?.description; @@ -87,12 +97,17 @@ export default function FieldDescription({ expandedField, editableFieldInfo }: P
Description - + + {isPropagated && } + {!!displayedDescription && ( + + )} +
{isSchemaEditable && ( )} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts new file mode 100644 index 0000000000000..6434baddb77a6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/getFieldDescriptionDetails.ts @@ -0,0 +1,25 @@ +import { EditableSchemaFieldInfo, SchemaFieldEntity } from '../../../../../../../types.generated'; + +interface Props { + schemaFieldEntity?: SchemaFieldEntity | null; + editableFieldInfo?: EditableSchemaFieldInfo; + defaultDescription?: string | null; +} + +export function getFieldDescriptionDetails({ schemaFieldEntity, editableFieldInfo, defaultDescription }: Props) { + const documentation = schemaFieldEntity?.documentation?.documentations?.[0]; + const isUsingDocumentationAspect = !editableFieldInfo?.description && !!documentation; + const isPropagated = + isUsingDocumentationAspect && + !!documentation?.attribution?.sourceDetail?.find( + (mapEntry) => mapEntry.key === 'propagated' && mapEntry.value === 'true', + ); + + const displayedDescription = + editableFieldInfo?.description || documentation?.documentation || defaultDescription || ''; + + const sourceDetail = documentation?.attribution?.sourceDetail; + const propagatedDescription = documentation?.documentation; + + return { displayedDescription, isPropagated, sourceDetail, propagatedDescription }; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx index 73e6d2ca6e9b3..bb70c2cb49303 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx @@ -6,6 +6,7 @@ import { useUpdateDescriptionMutation } from '../../../../../../../graphql/mutat import { useMutationUrn, useRefetch } from '../../../../EntityContext'; import { useSchemaRefetch } from '../SchemaContext'; import { pathMatchesNewPath } from '../../../../../dataset/profile/schema/utils/utils'; +import { getFieldDescriptionDetails } from './getFieldDescriptionDetails'; export default function useDescriptionRenderer(editableSchemaMetadata: EditableSchemaMetadata | null | undefined) { const urn = useMutationUrn(); @@ -21,10 +22,16 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS }; return (description: string, record: SchemaField, index: number): JSX.Element => { - const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find( - (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), + const editableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find((candidateEditableFieldInfo) => + pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), ); - const displayedDescription = relevantEditableFieldInfo?.description || description; + const { schemaFieldEntity } = record; + const { displayedDescription, isPropagated, sourceDetail } = getFieldDescriptionDetails({ + schemaFieldEntity, + editableFieldInfo, + defaultDescription: description, + }); + const sanitizedDescription = DOMPurify.sanitize(displayedDescription); const original = record.description ? DOMPurify.sanitize(record.description) : undefined; const businessAttributeDescription = @@ -43,7 +50,7 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS baExpanded={!!expandedBARows[index]} description={sanitizedDescription} original={original} - isEdited={!!relevantEditableFieldInfo?.description} + isEdited={!!editableFieldInfo?.description} onUpdate={(updatedDescription) => updateDescription({ variables: { @@ -56,6 +63,8 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS }, }).then(refresh) } + isPropagated={isPropagated} + sourceDetail={sourceDetail} isReadOnly /> ); diff --git a/datahub-web-react/src/app/entity/shared/useGetEntities.ts b/datahub-web-react/src/app/entity/shared/useGetEntities.ts new file mode 100644 index 0000000000000..9391bc17d7a8a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/useGetEntities.ts @@ -0,0 +1,18 @@ +import { useEffect, useState } from 'react'; +import { useGetEntitiesQuery } from '../../../graphql/entity.generated'; +import { Entity } from '../../../types.generated'; + +export function useGetEntities(urns: string[]): Entity[] { + const [verifiedUrns, setVerifiedUrns] = useState([]); + + useEffect(() => { + urns.forEach((urn) => { + if (urn.startsWith('urn:li:') && !verifiedUrns.includes(urn)) { + setVerifiedUrns((prevUrns) => [...prevUrns, urn]); + } + }); + }, [urns, verifiedUrns]); + + const { data } = useGetEntitiesQuery({ variables: { urns: verifiedUrns }, skip: !verifiedUrns.length }); + return (data?.entities || []) as Entity[]; +} diff --git a/smoke-test/tests/cypress/cypress.config.js b/smoke-test/tests/cypress/cypress.config.js index 7c3863ad869e3..cfc515766097d 100644 --- a/smoke-test/tests/cypress/cypress.config.js +++ b/smoke-test/tests/cypress/cypress.config.js @@ -2,25 +2,25 @@ const { defineConfig } = require("cypress"); module.exports = defineConfig({ - chromeWebSecurity: false, - viewportHeight: 960, - viewportWidth: 1536, - projectId: "hkrxk5", - defaultCommandTimeout: 10000, - retries: { - runMode: 2, - openMode: 0, - }, - video: false, - e2e: { - // We've imported your old cypress plugins here. - // You may want to clean this up later by importing these. - setupNodeEvents(on, config) { - // eslint-disable-next-line global-require - return require("./cypress/plugins/index")(on, config); - }, - baseUrl: "http://localhost:9002/", - specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", - experimentalStudio: true, - }, + chromeWebSecurity: false, + viewportHeight: 960, + viewportWidth: 1536, + projectId: "hkrxk5", + defaultCommandTimeout: 10000, + retries: { + runMode: 2, + openMode: 0, + }, + video: false, + e2e: { + // We've imported your old cypress plugins here. + // You may want to clean this up later by importing these. + setupNodeEvents(on, config) { + // eslint-disable-next-line global-require + return require("./cypress/plugins/index")(on, config); + }, + baseUrl: "http://localhost:9002/", + specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", + experimentalStudio: true, + } }); diff --git a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js new file mode 100644 index 0000000000000..49132bf53327c --- /dev/null +++ b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js @@ -0,0 +1,24 @@ +const testId = '[data-testid="docPropagationIndicator"]'; + +describe('docPropagation', () => { + it('logs in and navigates to the schema page and checks for docPropagationIndicator', () => { + cy.login(); + cy.visit('http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter='); + + // verify that the indicator exists in the table + cy.get(testId).should('exist'); + + // click on the table row + cy.get('[data-row-key="user_id"]').click(); + + // verify that the indicator exists in id="entity-profile-sidebar" + cy.get('[id="entity-profile-sidebar"]') + .then(($sidebar) => { + if ($sidebar.find(testId).length) return testId; + return null; + }) + .then((selector) => { + cy.get(selector).should('exist'); + }); + }) +}); \ No newline at end of file diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index 5253b7a33b085..b2df28934ce1c 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -1,2141 +1,2231 @@ [ - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/prod/kafka/SampleKafkaDataset"] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": null, - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleKafkaSchema", - "platform": "urn:li:dataPlatform:kafka", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=boolean].field_foo_2", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "globalTags": { - "tags": [{ "tag": "urn:li:tag:NeedsDocumentation" }] - }, - "recursive": false - }, - { - "fieldPath": "[version=2.0].[type=boolean].field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "[version=2.0].[key=True].[type=int].id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id specifying which partition the message should go to" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hdfs dataset", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/prod/hdfs/SampleCypressHdfsDataset"] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "editableSchemaFieldInfo": [ - { - "fieldPath": "shipment_info", - "globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] }, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHdfsSchema", - "platform": "urn:li:dataPlatform:hdfs", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "shipment_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.date", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info date description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "Date", - "recursive": false - }, - { - "fieldPath": "shipment_info.target", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info target description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false - }, - { - "fieldPath": "shipment_info.destination", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info destination description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lat", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lat" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lng", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lng" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { "tag": "urn:li:tag:Cypress" }, - { "tag": "urn:li:tag:Cypress2" } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hive dataset", - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false, - "isPartOfKey": true - }, - { - "fieldPath": "field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table where each row represents a single log event", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "event_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of your logging event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "event_data", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Data of your event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "TS the event was ingested" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "browser", - "jsonPath": null, - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day. Creted from log events.", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users deleted on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who was deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Timestamp user was deleted at" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false - }, - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "browser_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the browser" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "session_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the session" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "deletion_reason", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Why the user chose to deactivate" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - } - ], - "primaryKeys": ["user_name"], - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user session", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Creations", - "description": "Constructs the fct_users_created from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Deletions", - "description": "Constructs the fct_users_deleted from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "name": "Users", - "description": "Constructs the fct_users_deleted and fct_users_created tables", - "project": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 1", - "description": "Baz Chart 1", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 2", - "description": "Baz Chart 2", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": { - "array": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - } - ] - }, - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(looker,cypress_baz)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpGroup:bfoo", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "title": "Baz Dashboard", - "description": "Baz Dashboard", - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - }, - "charts": [ - "urn:li:chart:(looker,cypress_baz1)", - "urn:li:chart:(looker,cypress_baz2)" - ], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { - "urn": "urn:li:glossaryNode:CypressNode", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { - "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress", - "description": "Indicates the entity is for cypress integration test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress2", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress 2", - "description": "Indicates the entity is #2 for cypress test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:CypressFeatureTag" }] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "this is a description from source system", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:CypressPrimaryKeyTag" }] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/sagemaker/cypress-feature-table"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", - "creation_time": "2021-06-24 09:48:37.035000", - "status": "Created" - }, - "description": "Yet another test feature group", - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "customProperties": { - "EnableNetworkIsolation": "True" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", - "description": "ml model description", - "date": 1420070400000, - "version": null, - "type": null, - "hyperParameters": null, - "hyperParams": [ - { - "name": "parameter-1", - "description": null, - "value": "some-value", - "createdAt": null - } - ], - "trainingMetrics": [ - { - "name": "another-metric", - "description": null, - "value": "1.0", - "createdAt": null - } - ], - "onlineMetrics": null, - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "tags": [], - "deployments": [], - "trainingJobs": [], - "downstreamJobs": [], - "groups": [ - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/sagemaker/cypress-model-package-group/cypress-model"] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { - "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { - "customProperties": { - "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", - "ModelPackageGroupDescription": "Just a model package group.", - "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", - "ModelPackageGroupStatus": "Completed" - }, - "description": "Just a model package group.", - "createdAt": 1420070400000, - "version": null - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:some-user", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/sagemaker/cypress-model-package-group"] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a definition", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "domain", - "entityUrn": "urn:li:domain:marketing", - "changeType": "UPSERT", - "aspectName": "domainProperties", - "aspect": { - "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table for health", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionInfo", - "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "incident", - "entityUrn": "urn:li:incident:test", - "changeType": "UPSERT", - "aspectName": "incidentInfo", - "aspect": { - "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "changeType": "UPSERT", - "aspectName": "incidentsSummary", - "aspect": { - "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - } -] + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka/SampleKafkaDataset" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": null, + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleKafkaSchema", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=boolean].field_foo_2", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "recursive": false + }, + { + "fieldPath": "[version=2.0].[type=boolean].field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "[version=2.0].[key=True].[type=int].id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id specifying which partition the message should go to" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hdfs dataset", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/hdfs/SampleCypressHdfsDataset" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "editableSchemaFieldInfo": [ + { + "fieldPath": "shipment_info", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + }, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:hdfs", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "shipment_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.date", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info date description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "Date", + "recursive": false + }, + { + "fieldPath": "shipment_info.target", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info target description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false + }, + { + "fieldPath": "shipment_info.destination", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info destination description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lat", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lat" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lng", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lng" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + }, + { + "tag": "urn:li:tag:Cypress2" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hive dataset", + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "field_foo", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table where each row represents a single log event", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "event_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of your logging event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "event_data", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Data of your event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "TS the event was ingested" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "browser", + "jsonPath": null, + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day. Creted from log events.", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users deleted on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who was deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Timestamp user was deleted at" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "browser_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the browser" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "session_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the session" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "deletion_reason", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Why the user chose to deactivate" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + } + ], + "primaryKeys": [ + "user_name" + ], + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user session", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Creations", + "description": "Constructs the fct_users_created from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Deletions", + "description": "Constructs the fct_users_deleted from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "name": "Users", + "description": "Constructs the fct_users_deleted and fct_users_created tables", + "project": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 1", + "description": "Baz Chart 1", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 2", + "description": "Baz Chart 2", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": { + "array": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + } + ] + }, + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(looker,cypress_baz)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:bfoo", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "title": "Baz Dashboard", + "description": "Baz Dashboard", + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + }, + "charts": [ + "urn:li:chart:(looker,cypress_baz1)", + "urn:li:chart:(looker,cypress_baz2)" + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "dashboardUrl": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { + "urn": "urn:li:glossaryNode:CypressNode", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress", + "description": "Indicates the entity is for cypress integration test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress2", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress 2", + "description": "Indicates the entity is #2 for cypress test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "description": null, + "dataType": "TEXT", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressFeatureTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "description": "this is a description from source system", + "dataType": "ORDINAL", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressPrimaryKeyTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/cypress-feature-table" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", + "creation_time": "2021-06-24 09:48:37.035000", + "status": "Created" + }, + "description": "Yet another test feature group", + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": { + "EnableNetworkIsolation": "True" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", + "description": "ml model description", + "date": 1420070400000, + "version": null, + "type": null, + "hyperParameters": null, + "hyperParams": [ + { + "name": "parameter-1", + "description": null, + "value": "some-value", + "createdAt": null + } + ], + "trainingMetrics": [ + { + "name": "another-metric", + "description": null, + "value": "1.0", + "createdAt": null + } + ], + "onlineMetrics": null, + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "tags": [], + "deployments": [], + "trainingJobs": [], + "downstreamJobs": [], + "groups": [ + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/cypress-model-package-group/cypress-model" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { + "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { + "customProperties": { + "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", + "ModelPackageGroupDescription": "Just a model package group.", + "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", + "ModelPackageGroupStatus": "Completed" + }, + "description": "Just a model package group.", + "createdAt": 1420070400000, + "version": null + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:some-user", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/cypress-model-package-group" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a definition", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "domain", + "entityUrn": "urn:li:domain:marketing", + "changeType": "UPSERT", + "aspectName": "domainProperties", + "aspect": { + "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table for health", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "incident", + "entityUrn": "urn:li:incident:test", + "changeType": "UPSERT", + "aspectName": "incidentInfo", + "aspect": { + "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "changeType": "UPSERT", + "aspectName": "incidentsSummary", + "aspect": { + "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "schemaField", + "entityUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)", + "changeType": "UPSERT", + "aspectName": "documentation", + "aspect": { + "value": "{\"documentations\":[{\"attribution\":{\"actor\":\"urn:li:corpuser:__datahub_system\",\"source\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"sourceDetail\":{\"actor\":\"urn:li:corpuser:shirshanka@acryl.io\",\"origin\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"propagated\":\"true\"},\"time\":1721422917808},\"documentation\":\"Unique identifier of user profile.\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + } +] \ No newline at end of file From 15c2580ab670fcaab7bb1218559ea1bf8de904e0 Mon Sep 17 00:00:00 2001 From: Sam Black Date: Tue, 30 Jul 2024 16:58:55 -0400 Subject: [PATCH 06/18] Cypress prettier format fix --- smoke-test/tests/cypress/cypress.config.js | 42 +- .../cypress/e2e/actions/docPropagation.js | 40 +- smoke-test/tests/cypress/data.json | 4448 ++++++++--------- 3 files changed, 2260 insertions(+), 2270 deletions(-) diff --git a/smoke-test/tests/cypress/cypress.config.js b/smoke-test/tests/cypress/cypress.config.js index cfc515766097d..7c3863ad869e3 100644 --- a/smoke-test/tests/cypress/cypress.config.js +++ b/smoke-test/tests/cypress/cypress.config.js @@ -2,25 +2,25 @@ const { defineConfig } = require("cypress"); module.exports = defineConfig({ - chromeWebSecurity: false, - viewportHeight: 960, - viewportWidth: 1536, - projectId: "hkrxk5", - defaultCommandTimeout: 10000, - retries: { - runMode: 2, - openMode: 0, - }, - video: false, - e2e: { - // We've imported your old cypress plugins here. - // You may want to clean this up later by importing these. - setupNodeEvents(on, config) { - // eslint-disable-next-line global-require - return require("./cypress/plugins/index")(on, config); - }, - baseUrl: "http://localhost:9002/", - specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", - experimentalStudio: true, - } + chromeWebSecurity: false, + viewportHeight: 960, + viewportWidth: 1536, + projectId: "hkrxk5", + defaultCommandTimeout: 10000, + retries: { + runMode: 2, + openMode: 0, + }, + video: false, + e2e: { + // We've imported your old cypress plugins here. + // You may want to clean this up later by importing these. + setupNodeEvents(on, config) { + // eslint-disable-next-line global-require + return require("./cypress/plugins/index")(on, config); + }, + baseUrl: "http://localhost:9002/", + specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", + experimentalStudio: true, + }, }); diff --git a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js index 49132bf53327c..9cd449b3fce7c 100644 --- a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js +++ b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js @@ -1,24 +1,26 @@ const testId = '[data-testid="docPropagationIndicator"]'; -describe('docPropagation', () => { - it('logs in and navigates to the schema page and checks for docPropagationIndicator', () => { - cy.login(); - cy.visit('http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter='); +describe("docPropagation", () => { + it("logs in and navigates to the schema page and checks for docPropagationIndicator", () => { + cy.login(); + cy.visit( + "http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", + ); - // verify that the indicator exists in the table - cy.get(testId).should('exist'); + // verify that the indicator exists in the table + cy.get(testId).should("exist"); - // click on the table row - cy.get('[data-row-key="user_id"]').click(); + // click on the table row + cy.get('[data-row-key="user_id"]').click(); - // verify that the indicator exists in id="entity-profile-sidebar" - cy.get('[id="entity-profile-sidebar"]') - .then(($sidebar) => { - if ($sidebar.find(testId).length) return testId; - return null; - }) - .then((selector) => { - cy.get(selector).should('exist'); - }); - }) -}); \ No newline at end of file + // verify that the indicator exists in id="entity-profile-sidebar" + cy.get('[id="entity-profile-sidebar"]') + .then(($sidebar) => { + if ($sidebar.find(testId).length) return testId; + return null; + }) + .then((selector) => { + cy.get(selector).should("exist"); + }); + }); +}); diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index b2df28934ce1c..ce61f7c83a038 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -1,2231 +1,2219 @@ [ - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/kafka/SampleKafkaDataset" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": null, - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleKafkaSchema", - "platform": "urn:li:dataPlatform:kafka", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=boolean].field_foo_2", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:NeedsDocumentation" - } - ] - }, - "recursive": false - }, - { - "fieldPath": "[version=2.0].[type=boolean].field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "[version=2.0].[key=True].[type=int].id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id specifying which partition the message should go to" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hdfs dataset", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/hdfs/SampleCypressHdfsDataset" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "editableSchemaFieldInfo": [ - { - "fieldPath": "shipment_info", - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - }, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHdfsSchema", - "platform": "urn:li:dataPlatform:hdfs", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "shipment_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.date", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info date description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "Date", - "recursive": false - }, - { - "fieldPath": "shipment_info.target", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info target description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false - }, - { - "fieldPath": "shipment_info.destination", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info destination description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lat", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lat" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - }, - { - "fieldPath": "shipment_info.geo_info.lng", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lng" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - }, - { - "tag": "urn:li:tag:Cypress2" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "my hive dataset", - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false, - "isPartOfKey": true - }, - { - "fieldPath": "field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table where each row represents a single log event", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "event_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of your logging event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "event_data", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Data of your event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" - } - ], - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "TS the event was ingested" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "browser", - "jsonPath": null, - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day. Creted from log events.", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users deleted on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who was deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Timestamp user was deleted at" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false - }, - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "browser_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the browser" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "session_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the session" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "deletion_reason", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Why the user chose to deactivate" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - } - ], - "primaryKeys": [ - "user_name" - ], - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "user session", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Creations", - "description": "Constructs the fct_users_created from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Deletions", - "description": "Constructs the fct_users_deleted from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "name": "Users", - "description": "Constructs the fct_users_deleted and fct_users_created tables", - "project": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 1", - "description": "Baz Chart 1", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,cypress_baz2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 2", - "description": "Baz Chart 2", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": { - "array": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - } - ] - }, - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(looker,cypress_baz)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpGroup:bfoo", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "title": "Baz Dashboard", - "description": "Baz Dashboard", - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - }, - "charts": [ - "urn:li:chart:(looker,cypress_baz1)", - "urn:li:chart:(looker,cypress_baz2)" - ], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:Cypress" - } - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { - "urn": "urn:li:glossaryNode:CypressNode", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { - "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress", - "description": "Indicates the entity is for cypress integration test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Cypress2", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Cypress 2", - "description": "Indicates the entity is #2 for cypress test purposes" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:CypressFeatureTag" - } - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "this is a description from source system", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:CypressPrimaryKeyTag" - } - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-feature-table" - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", - "creation_time": "2021-06-24 09:48:37.035000", - "status": "Created" - }, - "description": "Yet another test feature group", - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "customProperties": { - "EnableNetworkIsolation": "True" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", - "description": "ml model description", - "date": 1420070400000, - "version": null, - "type": null, - "hyperParameters": null, - "hyperParams": [ - { - "name": "parameter-1", - "description": null, - "value": "some-value", - "createdAt": null - } - ], - "trainingMetrics": [ - { - "name": "another-metric", - "description": null, - "value": "1.0", - "createdAt": null - } - ], - "onlineMetrics": null, - "mlFeatures": [ - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" - ], - "tags": [], - "deployments": [], - "trainingJobs": [], - "downstreamJobs": [], - "groups": [ - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-model-package-group/cypress-model" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { - "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { - "customProperties": { - "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", - "ModelPackageGroupDescription": "Just a model package group.", - "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", - "ModelPackageGroupStatus": "Completed" - }, - "description": "Just a model package group.", - "createdAt": 1420070400000, - "version": null - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:some-user", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-model-package-group" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", - "aspects": [ - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a definition", - "parentNode": "urn:li:glossaryNode:CypressNode", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "domain", - "entityUrn": "urn:li:domain:marketing", - "changeType": "UPSERT", - "aspectName": "domainProperties", - "aspect": { - "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "query", - "entityUrn": "urn:li:query:test-query", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table for health", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionInfo", - "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "assertionRunEvent", - "aspect": { - "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "incident", - "entityUrn": "urn:li:incident:test", - "changeType": "UPSERT", - "aspectName": "incidentInfo", - "aspect": { - "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", - "changeType": "UPSERT", - "aspectName": "incidentsSummary", - "aspect": { - "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "businessAttributeInfo", - "aspect": { - "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "businessAttribute", - "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", - "contentType": "application/json" - }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "schemaField", - "entityUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)", - "changeType": "UPSERT", - "aspectName": "documentation", - "aspect": { - "value": "{\"documentations\":[{\"attribution\":{\"actor\":\"urn:li:corpuser:__datahub_system\",\"source\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"sourceDetail\":{\"actor\":\"urn:li:corpuser:shirshanka@acryl.io\",\"origin\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"propagated\":\"true\"},\"time\":1721422917808},\"documentation\":\"Unique identifier of user profile.\"}]}", - "contentType": "application/json" - }, - "systemMetadata": null - } -] \ No newline at end of file + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/prod/kafka/SampleKafkaDataset"] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": null, + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleKafkaSchema", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=boolean].field_foo_2", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "recursive": false + }, + { + "fieldPath": "[version=2.0].[type=boolean].field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "[version=2.0].[key=True].[type=int].id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id specifying which partition the message should go to" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hdfs dataset", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/prod/hdfs/SampleCypressHdfsDataset"] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "editableSchemaFieldInfo": [ + { + "fieldPath": "shipment_info", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + }, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:hdfs", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "shipment_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.date", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info date description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "Date", + "recursive": false + }, + { + "fieldPath": "shipment_info.target", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info target description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false + }, + { + "fieldPath": "shipment_info.destination", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info destination description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lat", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lat" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + }, + { + "fieldPath": "shipment_info.geo_info.lng", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Shipment info geo_info lng" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + }, + { + "tag": "urn:li:tag:Cypress2" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "my hive dataset", + "uri": null, + "tags": [], + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "field_foo", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "field_bar", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Bar field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table where each row represents a single log event", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "event_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of your logging event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "event_data", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Data of your event" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "TS the event was ingested" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + }, + { + "fieldPath": "browser", + "jsonPath": null, + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day. Creted from log events.", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users created on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user created" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who signed up" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table containing all the users deleted on a single day", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "user_name", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Name of the user who was deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Timestamp user was deleted at" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Id of the user deleted" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "browser_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the browser" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "session_id", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Cookie attached to identify the session" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "fieldPath": "deletion_reason", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Why the user chose to deactivate" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + } + ], + "primaryKeys": ["user_name"], + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "user session", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Creations", + "description": "Constructs the fct_users_created from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Deletions", + "description": "Constructs the fct_users_deleted from logging_events", + "type": "SQL", + "flowUrn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(airflow,cypress_dag_abc,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "name": "Users", + "description": "Constructs the fct_users_deleted and fct_users_created tables", + "project": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 1", + "description": "Baz Chart 1", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,cypress_baz2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "title": "Baz Chart 2", + "description": "Baz Chart 2", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": { + "array": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + } + ] + }, + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(looker,cypress_baz)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:bfoo", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "title": "Baz Dashboard", + "description": "Baz Dashboard", + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + }, + "charts": [ + "urn:li:chart:(looker,cypress_baz1)", + "urn:li:chart:(looker,cypress_baz2)" + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "deleted": null + }, + "dashboardUrl": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Cypress" + } + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressTerm", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { + "urn": "urn:li:glossaryNode:CypressNode", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress", + "description": "Indicates the entity is for cypress integration test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Cypress2", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Cypress 2", + "description": "Indicates the entity is #2 for cypress test purposes" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "value": "{\"customProperties\": {\"run_id\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"duration\": \"None\", \"start_date\": \"2022-03-30 11:35:09.761704+00:00\", \"end_date\": \"None\", \"execution_date\": \"2022-03-30 11:35:08.970522+00:00\", \"try_number\": \"0\", \"hostname\": \"e4688afa8c61\", \"max_tries\": \"0\", \"external_executor_id\": \"None\", \"pid\": \"44744\", \"state\": \"running\", \"operator\": \"PythonOperator\", \"priority_weight\": \"1\", \"unixname\": \"airflow\", \"log_url\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\"}, \"externalUrl\": \"http://localhost:8080/log?execution_date=2022-03-30T11%3A35%3A08.970522%2B00%3A00&task_id=ingest_airflow_dags&dag_id=datahub_airflow_ingest\", \"name\": \"manual__2022-03-30T11:35:08.970522+00:00\", \"type\": \"BATCH_AD_HOC\", \"created\": {\"time\": 1648630100320, \"actor\": \"urn:li:corpuser:datahub\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "value": "{\"inputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "value": "{\"outputs\": [\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)\"]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "value": "{\"parentTemplate\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)\", \"upstreamInstances\": []}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1648640109761, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResultType\": \"failed\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "description": null, + "dataType": "TEXT", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressFeatureTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "description": "this is a description from source system", + "dataType": "ORDINAL", + "version": null, + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:CypressPrimaryKeyTag" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/sagemaker/cypress-feature-table"] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", + "creation_time": "2021-06-24 09:48:37.035000", + "status": "Created" + }, + "description": "Yet another test feature group", + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": { + "EnableNetworkIsolation": "True" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", + "description": "ml model description", + "date": 1420070400000, + "version": null, + "type": null, + "hyperParameters": null, + "hyperParams": [ + { + "name": "parameter-1", + "description": null, + "value": "some-value", + "createdAt": null + } + ], + "trainingMetrics": [ + { + "name": "another-metric", + "description": null, + "value": "1.0", + "createdAt": null + } + ], + "onlineMetrics": null, + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], + "tags": [], + "deployments": [], + "trainingJobs": [], + "downstreamJobs": [], + "groups": [ + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/sagemaker/cypress-model-package-group/cypress-model"] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { + "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { + "customProperties": { + "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", + "ModelPackageGroupDescription": "Just a model package group.", + "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", + "ModelPackageGroupStatus": "Completed" + }, + "description": "Just a model package group.", + "createdAt": 1420070400000, + "version": null + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:some-user", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": ["/sagemaker/cypress-model-package-group"] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "definition": "a definition", + "parentNode": "urn:li:glossaryNode:CypressNode", + "sourceRef": "FIBO", + "termSource": "EXTERNAL", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", + "customProperties": { + "FQDN": "SavingAccount" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "domain", + "entityUrn": "urn:li:domain:marketing", + "changeType": "UPSERT", + "aspectName": "domainProperties", + "aspect": { + "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT*FROMTABLE\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "query", + "entityUrn": "urn:li:query:test-query", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "description": "table for health", + "uri": null, + "tags": [], + "customProperties": { + "encoding": "utf-8" + } + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1675155843000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "incident", + "entityUrn": "urn:li:incident:test", + "changeType": "UPSERT", + "aspectName": "incidentInfo", + "aspect": { + "value": "{\"entities\":[\"urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)\"],\"created\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854918},\"description\":\"test\",\"source\":{\"type\":\"MANUAL\"},\"title\":\"test\",\"type\":\"OPERATIONAL\",\"status\":{\"lastUpdated\":{\"actor\":\"urn:li:corpuser:admin\",\"time\":1692301854919},\"state\":\"ACTIVE\"}}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)", + "changeType": "UPSERT", + "aspectName": "incidentsSummary", + "aspect": { + "value": "{\"resolvedIncidentDetails\":[], \"activeIncidentDetails\": [{ \"urn\": \"urn:li:incident:test\", \"type\": \"OPERATIONAL\", \"createdAt\": 0 }]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "schemaField", + "entityUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)", + "changeType": "UPSERT", + "aspectName": "documentation", + "aspect": { + "value": "{\"documentations\":[{\"attribution\":{\"actor\":\"urn:li:corpuser:__datahub_system\",\"source\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"sourceDetail\":{\"actor\":\"urn:li:corpuser:shirshanka@acryl.io\",\"origin\":\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)\",\"propagated\":\"true\"},\"time\":1721422917808},\"documentation\":\"Unique identifier of user profile.\"}]}", + "contentType": "application/json" + }, + "systemMetadata": null + } +] From 23eba7777bed3556e3cbecaf51ca6c65d7c701c1 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 30 Jul 2024 18:16:06 -0700 Subject: [PATCH 07/18] Adding some fluff --- .../components/SchemaDescriptionField.tsx | 38 +++++------ .../shared/propagation/PropagationDetails.tsx | 64 ++++++++++--------- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx index e30cd1405ef9b..e7d986028d4a6 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx @@ -195,29 +195,29 @@ export default function DescriptionField({ ) : ( <> - {/* - // { - // e.stopPropagation(); - // handleExpanded(true); - // }} - // > - // Read More - // - // - // } - suffix={EditButton} - shouldWrap - > */} {isPropagated && }   - {description} + + { + e.stopPropagation(); + handleExpanded(true); + }} + > + Read More + + + } + suffix={EditButton} + shouldWrap + > + {description} + - {/* */} )} {isEdited && (edited)} diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index 1a2f9de10ce72..18ea438d7fc30 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -1,33 +1,29 @@ import React from 'react'; import styled from 'styled-components'; -import { ThunderboltOutlined } from '@ant-design/icons'; import { Popover } from 'antd'; import { StringMapEntry } from '../../../../types.generated'; import PropagationEntityLink from './PropagationEntityLink'; import { usePropagationDetails } from './utils'; +import { ANTD_GRAY } from '../constants'; +import { PropagateThunderbolt, PropagateThunderboltFilled } from './PropagationIcon'; const PopoverWrapper = styled.div` display: flex; flex-direction: column; - gap: 4px; `; -const PropagateThunderbolt = styled(ThunderboltOutlined)` - color: #8088a3; +const PopoverTitle = styled.div` font-weight: bold; - &:hover { - color: #4b39bc; - } + font-size: 14px; + padding: 6px 0px; `; -const EntityWrapper = styled.span` - display: inline-flex; - align-items: center; - gap: 2px; -`; - -const PropagationPrefix = styled.span` - color: black; +const PopoverDescription = styled.div` + max-width: 340px; + font-size: 14px; + color: ${ANTD_GRAY[7]}; + display: inline; + padding: 0px 0px 4px 0px; `; interface Props { @@ -46,25 +42,33 @@ export default function PropagationDetails({ sourceDetail }: Props) { const popoverContent = originEntity || viaEntity ? ( - Propagated description -
- {viaEntity && ( - - via: - - - )} - {originEntity && originEntity.urn !== viaEntity?.urn && ( - - origin: - - - )} + + This description was automatically propagated{' '} + {originEntity && originEntity.urn !== viaEntity?.urn && ( + <> + from + + )} + {viaEntity && ( + <> + via + + )} + ) : undefined; return ( - + + + Propagated Description + + } + content={popoverContent} + > ); From 907cba4cb9fa61cd29b1a82f4c8c04ddd9c28091 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 30 Jul 2024 18:16:41 -0700 Subject: [PATCH 08/18] adding propagation action --- .../shared/propagation/PropagationIcon.tsx | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx new file mode 100644 index 0000000000000..01b4570c4ca0d --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationIcon.tsx @@ -0,0 +1,22 @@ +import styled from 'styled-components'; +import { ThunderboltFilled } from '@ant-design/icons'; +import { REDESIGN_COLORS } from '../constants'; + +export const PropagateThunderbolt = styled(ThunderboltFilled)` + && { + color: #a7c7fa; + } + font-size: 16px; + &:hover { + color: ${REDESIGN_COLORS.BLUE}; + } + margin-right: 4px; +`; + +export const PropagateThunderboltFilled = styled(ThunderboltFilled)` + && { + color: ${REDESIGN_COLORS.BLUE}; + } + font-size: 16px; + margin-right: 4px; +`; From 486f7bb770f623215388e1a0300ffdf8f4b2cfeb Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 30 Jul 2024 20:42:37 -0700 Subject: [PATCH 09/18] Adding --- .../src/app/settings/features/Feature.tsx | 33 ++++++++++--------- .../src/app/settings/features/Features.tsx | 2 +- .../cypress/e2e/actions/docPropagation.js | 2 +- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/datahub-web-react/src/app/settings/features/Feature.tsx b/datahub-web-react/src/app/settings/features/Feature.tsx index 2c090aae696f8..3bb1411145abe 100644 --- a/datahub-web-react/src/app/settings/features/Feature.tsx +++ b/datahub-web-react/src/app/settings/features/Feature.tsx @@ -134,22 +134,6 @@ export const Feature = ({ key, title, description, settings, options, isNew, lea - {settings.map((option) => ( - <> - - - - {option.title} - - - - - - - - ))} {options.map((option, index) => ( <> @@ -175,5 +159,22 @@ export const Feature = ({ key, title, description, settings, options, isNew, lea ))} + {settings.map((option) => ( + <> + + + + {option.title} + Only available on DataHub Cloud + + + + + + + + ))} ); diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx index ee8d7c628c1ef..01fdb0d42ffeb 100644 --- a/datahub-web-react/src/app/settings/features/Features.tsx +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -79,7 +79,7 @@ export const Features = () => { title: 'Asset Level Propagation', description: 'Propagate new documentation from upstream to downstream assets based on data lineage relationships.', - isAvailable: false, + isAvailable: true, checked: false, }, ], diff --git a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js index 9cd449b3fce7c..c578a35946cd2 100644 --- a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js +++ b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js @@ -4,7 +4,7 @@ describe("docPropagation", () => { it("logs in and navigates to the schema page and checks for docPropagationIndicator", () => { cy.login(); cy.visit( - "http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", ); // verify that the indicator exists in the table From 3b3ef053b1af83f140d00f1b10263671f441ec66 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 31 Jul 2024 09:00:33 -0700 Subject: [PATCH 10/18] Update docPropagation.js --- smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js index a84c3f1c69058..3d7e14195ab64 100644 --- a/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js +++ b/smoke-test/tests/cypress/cypress/e2e/actions/docPropagation.js @@ -4,7 +4,7 @@ describe("docPropagation", () => { it("logs in and navigates to the schema page and checks for docPropagationIndicator", () => { cy.login(); cy.visit( - "http://localhost:3000/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)/Schema?is_lineage_mode=false&schemaFilter=", ); From 0598921f4b68e42dced8b73a5ae7bc6633673cc1 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 31 Jul 2024 11:13:38 -0700 Subject: [PATCH 11/18] Update DescriptionModal.tsx --- .../app/entity/shared/components/legacy/DescriptionModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx index 8d14ab6ded312..2797606e93831 100644 --- a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx @@ -77,7 +77,7 @@ export default function UpdateDescriptionModal({ >
- + {!isAddDesc && description && original && ( Original:}> From 1c86fdd632593006c5f4d9c7ce083e8ea01af99e Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 31 Jul 2024 13:50:58 -0700 Subject: [PATCH 12/18] Adding propagation details fixes --- .../shared/propagation/PropagationDetails.tsx | 48 ++++++++++++++----- .../propagation/PropagationEntityLink.tsx | 8 +++- .../src/app/lineage/LineageExplorer.tsx | 2 +- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index 18ea438d7fc30..e887c29cc4165 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -4,7 +4,6 @@ import { Popover } from 'antd'; import { StringMapEntry } from '../../../../types.generated'; import PropagationEntityLink from './PropagationEntityLink'; import { usePropagationDetails } from './utils'; -import { ANTD_GRAY } from '../constants'; import { PropagateThunderbolt, PropagateThunderboltFilled } from './PropagationIcon'; const PopoverWrapper = styled.div` @@ -16,14 +15,37 @@ const PopoverTitle = styled.div` font-weight: bold; font-size: 14px; padding: 6px 0px; + color: #EEECFA; `; const PopoverDescription = styled.div` max-width: 340px; font-size: 14px; - color: ${ANTD_GRAY[7]}; + color: #EEECFA; display: inline; - padding: 0px 0px 4px 0px; + padding: 0px 0px 8px 0px; +`; + +const PopoverAttributes = styled.div` + display: flex; +`; + +const PopoverAttribute = styled.div` + margin-right: 12px; + margin-bottom: 4px; +`; + +const PopoverAttributeTitle = styled.div` + font-size: 14px; + color: #EEECFA; + font-weight: bold; + margin: 8px 0px; + overflow: hidden; + text-overflow: ellipsis; +`; + +const PopoverDocumentation = styled.a` + margin-top: 12px; `; interface Props { @@ -43,23 +65,27 @@ export default function PropagationDetails({ sourceDetail }: Props) { originEntity || viaEntity ? ( - This description was automatically propagated{' '} + This description was automatically propagated from an upstream column. Learn more + + + {originEntity && originEntity.urn !== viaEntity?.urn && ( - <> - from - + + Origin + )} {viaEntity && ( - <> - via - + + Via + )} - + ) : undefined; return ( diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx index b41fb51918837..6ccd54b5bc769 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx @@ -11,12 +11,16 @@ const PreviewImage = styled.img<{ size: number }>` min-width: ${(props) => props.size}px; object-fit: contain; background-color: transparent; - margin: 0 3px; + margin: 0px 4px 0px 0px; `; const StyledLink = styled(Link)` + margin-right: 4px; display: flex; align-items: center; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; `; interface Props { @@ -42,9 +46,11 @@ export default function PropagationEntityLink({ entity }: Props) { } return ( + <> {entityDisplayName} + ); } diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx index 26ffaa26a6ca2..0d435b959a186 100644 --- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx @@ -221,7 +221,7 @@ export default function LineageExplorer({ urn, type }: Props) { Close {selectedEntity.type !== EntityType.Restricted && ( - )} From 215a31a50b9bd50e2a35ec41136b8dc165f2334c Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 9 Aug 2024 12:52:10 -0700 Subject: [PATCH 13/18] pushing lint fix --- .../graphql/types/chart/mappers/InputFieldsMapper.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java index a4e40750f0d65..269fb7d4ddf79 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java @@ -11,7 +11,6 @@ import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; - @Slf4j public class InputFieldsMapper { @@ -40,9 +39,14 @@ public com.linkedin.datahub.graphql.generated.InputFields apply( if (field.hasSchemaFieldUrn()) { fieldResult.setSchemaFieldUrn(field.getSchemaFieldUrn().toString()); try { - parentUrn = Urn.createFromString(field.getSchemaFieldUrn().getEntityKey().get(0)); + parentUrn = + Urn.createFromString(field.getSchemaFieldUrn().getEntityKey().get(0)); } catch (URISyntaxException e) { - log.error("Field urn resolution: failed to extract parentUrn successfully from {}. Falling back to {}", field.getSchemaFieldUrn(), entityUrn, e); + log.error( + "Field urn resolution: failed to extract parentUrn successfully from {}. Falling back to {}", + field.getSchemaFieldUrn(), + entityUrn, + e); } } if (field.hasSchemaField()) { From bf3b73992d5a63332224acb368de7880cbc475ab Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 9 Aug 2024 15:09:29 -0700 Subject: [PATCH 14/18] Adding lint fixes --- datahub-web-react/.eslintrc.js | 2 +- datahub-web-react/README.md | 130 +++++++++--------- .../components/legacy/DescriptionModal.tsx | 7 +- .../shared/propagation/PropagationDetails.tsx | 20 +-- .../propagation/PropagationEntityLink.tsx | 8 +- .../src/app/lineage/LineageExplorer.tsx | 4 +- 6 files changed, 92 insertions(+), 79 deletions(-) diff --git a/datahub-web-react/.eslintrc.js b/datahub-web-react/.eslintrc.js index 5627283af1af1..3fdf7b6a3042c 100644 --- a/datahub-web-react/.eslintrc.js +++ b/datahub-web-react/.eslintrc.js @@ -48,7 +48,7 @@ module.exports = { ], 'vitest/prefer-to-be': 'off', '@typescript-eslint/no-use-before-define': ['error', { functions: false, classes: false }], - 'react-refresh/only-export-components': ['warn', { 'allowConstantExport': true }], + 'react-refresh/only-export-components': ['warn', { allowConstantExport: true }], }, settings: { react: { diff --git a/datahub-web-react/README.md b/datahub-web-react/README.md index 560f5315b2c71..86bbb349b027c 100644 --- a/datahub-web-react/README.md +++ b/datahub-web-react/README.md @@ -1,44 +1,47 @@ --- -title: "datahub-web-react" +title: 'datahub-web-react' --- # DataHub React App ## About -This module contains a React application that serves as the DataHub UI. -Feel free to take a look around, deploy, and contribute. +This module contains a React application that serves as the DataHub UI. +Feel free to take a look around, deploy, and contribute. ## Functional Goals + The initial milestone for the app was to achieve functional parity with the previous Ember app. This meant supporting -- Dataset Profiles, Search, Browse Experience -- User Profiles, Search -- LDAP Authentication Flow +- Dataset Profiles, Search, Browse Experience +- User Profiles, Search +- LDAP Authentication Flow -This has since been achieved. The new set of functional goals are reflected in the latest version of the [DataHub Roadmap](../docs/roadmap.md). +This has since been achieved. The new set of functional goals are reflected in the latest version of the [DataHub Roadmap](../docs/roadmap.md). ## Design Goals + In building out the client experience, we intend to leverage learnings from the previous Ember-based app and incorporate feedback gathered from organizations operating DataHub. Two themes have emerged to serve as guideposts: -1. **Configurability**: The client experience should be configurable, such that deploying organizations can tailor certain - aspects to their needs. This includes theme / styling configurability, showing and hiding specific functionality, - customizing copy & logos, etc. - -2. **Extensibility**: Extending the *functionality* of DataHub should be as simple as possible. Making changes like - extending an existing entity & adding a new entity should require minimal effort and should be well covered in detailed - documentation. +1. **Configurability**: The client experience should be configurable, such that deploying organizations can tailor certain + aspects to their needs. This includes theme / styling configurability, showing and hiding specific functionality, + customizing copy & logos, etc. +2. **Extensibility**: Extending the _functionality_ of DataHub should be as simple as possible. Making changes like + extending an existing entity & adding a new entity should require minimal effort and should be well covered in detailed + documentation. ## Starting the Application ### Quick Start Navigate to the `docker` directory and run the following to spin up the react app: + ``` ./quickstart.sh ``` + at `http://localhost:9002`. If you want to make changes to the UI see them live without having to rebuild the `datahub-frontend-react` docker image, you @@ -54,8 +57,9 @@ Optionally you could also start the app with the mock server without running the ### Testing your customizations There is two options to test your customizations: -* **Option 1**: Initialize the docker containers with the `quickstart.sh` script (or if any custom docker-compose file) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at `http://localhost:9002` to fetch real data. -* **Option 2**: Change the environment variable `REACT_APP_PROXY_TARGET` in the `.env` file to point to your `datahub-frontend` server (ex: https://my_datahub_host.com) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at some domain to fetch real data. + +- **Option 1**: Initialize the docker containers with the `quickstart.sh` script (or if any custom docker-compose file) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at `http://localhost:9002` to fetch real data. +- **Option 2**: Change the environment variable `REACT_APP_PROXY_TARGET` in the `.env` file to point to your `datahub-frontend` server (ex: https://my_datahub_host.com) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at some domain to fetch real data. The option 2 is useful if you want to test your React customizations without having to run the hole DataHub stack locally. However, if you changed other components of the DataHub stack, you will need to run the hole stack locally (building the docker images) and use the option 1. @@ -68,10 +72,10 @@ In order to start a server and run frontend unit tests using react-testing-frame There are also more automated tests using Cypress in the `smoke-test` folder of the repository root. #### Troubleshooting + `Error: error:0308010C:digital envelope routines::unsupported`: This error message shows up when using Node 17, due to an OpenSSL update related to md5. The best workaround is to revert to the Active LTS version of Node, 16.13.0 with the command `nvm install 16.13.0` and if necessary reinstall yarn `npm install --global yarn`. - ### Theming #### Customizing your App without rebuilding assets @@ -108,74 +112,74 @@ you to terminate and re-run `yarn start` to see updated styles. The `src` dir of the app is broken down into the following modules -**conf** - Stores global configuration flags that can be referenced across the app. For example, the number of +**conf** - Stores global configuration flags that can be referenced across the app. For example, the number of search results shown per page, or the placeholder text in the search bar box. It serves as a location where levels -for functional configurability should reside. +for functional configurability should reside. **app** - Contains all important components of the app. It has a few sub-modules: -- `auth`: Components used to render the user authentication experience. -- `browse`: Shared components used to render the 'browse-by-path' experience. The experience is akin to navigating a filesystem hierarchy. -- `preview`: Shared components used to render Entity 'preview' views. These can appear in search results, browse results, - and within entity profile pages. -- `search`: Shared components used to render the full-text search experience. -- `shared`: Misc. shared components -- `entity`: Contains Entity definitions, where entity-specific functionality resides. - Configuration is provided by implementing the 'Entity' interface. (See DatasetEntity.tsx for example) - There are 2 visual components each entity should supply: - - `profiles`: display relevant details about an individual entity. This serves as the entity's 'profile'. - - `previews`: provide a 'preview', or a smaller details card, containing the most important information about an entity instance. - - When rendering a preview, the entity's data and the type of preview (SEARCH, BROWSE, PREVIEW) are provided. This +- `auth`: Components used to render the user authentication experience. +- `browse`: Shared components used to render the 'browse-by-path' experience. The experience is akin to navigating a filesystem hierarchy. +- `preview`: Shared components used to render Entity 'preview' views. These can appear in search results, browse results, + and within entity profile pages. +- `search`: Shared components used to render the full-text search experience. +- `shared`: Misc. shared components +- `entity`: Contains Entity definitions, where entity-specific functionality resides. + Configuration is provided by implementing the 'Entity' interface. (See DatasetEntity.tsx for example) + There are 2 visual components each entity should supply: + + - `profiles`: display relevant details about an individual entity. This serves as the entity's 'profile'. + - `previews`: provide a 'preview', or a smaller details card, containing the most important information about an entity instance. + + When rendering a preview, the entity's data and the type of preview (SEARCH, BROWSE, PREVIEW) are provided. This allows you to optionally customize the way an entities preview is rendered in different views. - - - `entity registry`: There's another very important piece of code living within this module: the **EntityRegistry**. This is a layer + + - `entity registry`: There's another very important piece of code living within this module: the **EntityRegistry**. This is a layer of abstraction over the intimate details of rendering a particular entity. It is used to render a view associated with a particular entity type (user, dataset, etc.). - - +

-**graphql** - The React App talks to the `dathub-frontend` server using GraphQL. This module is where the *queries* issued -against the server are defined. Once defined, running `yarn run generate` will code-gen TypeScript objects to make invoking +**graphql** - The React App talks to the `dathub-frontend` server using GraphQL. This module is where the _queries_ issued +against the server are defined. Once defined, running `yarn run generate` will code-gen TypeScript objects to make invoking these queries extremely easy. An example can be found at the top of `SearchPage.tsx.` -**images** - Images to be displayed within the app. This is where one would place a custom logo image. +**images** - Images to be displayed within the app. This is where one would place a custom logo image. ## Adding an Entity The following outlines a series of steps required to introduce a new entity into the React app: -1. Declare the GraphQL Queries required to display the new entity - - If search functionality should be supported, extend the "search" query within `search.graphql` to fetch the new +1. Declare the GraphQL Queries required to display the new entity + + - If search functionality should be supported, extend the "search" query within `search.graphql` to fetch the new + entity data. + - If browse functionality should be supported, extend the "browse" query within `browse.graphql` to fetch the new entity data. - - If browse functionality should be supported, extend the "browse" query within `browse.graphql` to fetch the new - entity data. - - If display a 'profile' should be supported (most often), introduce a new `.graphql` file that contains a - `get` query to fetch the entity by primary key (urn). - - Note that your new entity *must* implement the `Entity` GraphQL type interface, and thus must have a corresponding - `EntityType`. - - -2. Implement the `Entity` interface + - If display a 'profile' should be supported (most often), introduce a new `.graphql` file that contains a + `get` query to fetch the entity by primary key (urn). + + Note that your new entity _must_ implement the `Entity` GraphQL type interface, and thus must have a corresponding + `EntityType`. + +2. Implement the `Entity` interface + - Create a new folder under `src/components/entity` corresponding to your entity - Create a class that implements the `Entity` interface (example: `DatasetEntity.tsx`) - - Provide an implementation each method defined on the interface. - - This class specifies whether your new entity should be searchable & browsable, defines the names used to - identify your entity when instances are rendered in collection / when entity appears - in the URL path, and provides the ability to render your entity given data returned by the GQL API. - + - Provide an implementation each method defined on the interface. + - This class specifies whether your new entity should be searchable & browsable, defines the names used to + identify your entity when instances are rendered in collection / when entity appears + in the URL path, and provides the ability to render your entity given data returned by the GQL API. 3. Register the new entity in the `EntityRegistry` - - Update `App.tsx` to register an instance of your new entity. Now your entity will be accessible via the registry + - Update `App.tsx` to register an instance of your new entity. Now your entity will be accessible via the registry and appear in the UI. To manually retrieve the info about your entity or others, simply use an instance - of the `EntityRegistry`, which is provided via `ReactContext` to *all* components in the hierarchy. + of the `EntityRegistry`, which is provided via `ReactContext` to _all_ components in the hierarchy. For example - ``` - entityRegistry.getCollectionName(EntityType.YOUR_NEW_ENTITY) - ``` - -That's it! For any questions, do not hesitate to reach out on the DataHub Slack community in #datahub-react. + ``` + entityRegistry.getCollectionName(EntityType.YOUR_NEW_ENTITY) + ``` + +That's it! For any questions, do not hesitate to reach out on the DataHub Slack community in #datahub-react. diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx index 2797606e93831..2d65a305b4cc8 100644 --- a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx @@ -77,7 +77,12 @@ export default function UpdateDescriptionModal({ > - + {!isAddDesc && description && original && ( Original:}> diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index e887c29cc4165..2c3fd4430fa42 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -15,19 +15,19 @@ const PopoverTitle = styled.div` font-weight: bold; font-size: 14px; padding: 6px 0px; - color: #EEECFA; + color: #eeecfa; `; const PopoverDescription = styled.div` max-width: 340px; font-size: 14px; - color: #EEECFA; + color: #eeecfa; display: inline; padding: 0px 0px 8px 0px; `; const PopoverAttributes = styled.div` - display: flex; + display: flex; `; const PopoverAttribute = styled.div` @@ -37,7 +37,7 @@ const PopoverAttribute = styled.div` const PopoverAttributeTitle = styled.div` font-size: 14px; - color: #EEECFA; + color: #eeecfa; font-weight: bold; margin: 8px 0px; overflow: hidden; @@ -65,18 +65,20 @@ export default function PropagationDetails({ sourceDetail }: Props) { originEntity || viaEntity ? ( - This description was automatically propagated from an upstream column. Learn more - + This description was automatically propagated from an upstream column.{' '} + Learn more {originEntity && originEntity.urn !== viaEntity?.urn && ( - Origin + Origin + )} {viaEntity && ( - Via + Via + )} @@ -85,7 +87,7 @@ export default function PropagationDetails({ sourceDetail }: Props) { return ( diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx index 6ccd54b5bc769..8c1285dd5808b 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx @@ -47,10 +47,10 @@ export default function PropagationEntityLink({ entity }: Props) { return ( <> - - - {entityDisplayName} - + + + {entityDisplayName} + ); } diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx index 0d435b959a186..ce0c4bb8f122d 100644 --- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx @@ -221,7 +221,9 @@ export default function LineageExplorer({ urn, type }: Props) { Close {selectedEntity.type !== EntityType.Restricted && ( - )} From d1487bc4960051815f4cdf306cca7274ef0bfd5e Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 9 Aug 2024 17:00:03 -0700 Subject: [PATCH 15/18] Final docs and refactor --- .../DocPropagationSettingsResolver.java | 3 +- .../shared/propagation/PropagationDetails.tsx | 2 +- .../src/app/settings/SettingsPage.tsx | 2 +- .../src/app/settings/features/Feature.tsx | 12 +- .../src/app/settings/features/Features.tsx | 3 +- docs-website/sidebars.js | 17 +++ docs/automation/docs-propagation.md | 128 ++++++++++++++++++ docs/automation/snowflake-tag-propagation.md | 88 ++++++++++++ .../settings/global/GlobalSettingsInfo.pdl | 5 +- .../main/resources/boot/global_settings.json | 4 + 10 files changed, 253 insertions(+), 11 deletions(-) create mode 100644 docs/automation/docs-propagation.md create mode 100644 docs/automation/snowflake-tag-propagation.md diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java index 84d3bcd7b376c..389a9263e7cfa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java @@ -33,7 +33,8 @@ public CompletableFuture get(final DataFetchingEnvironme final GlobalSettingsInfo globalSettings = _settingsService.getGlobalSettings(context.getOperationContext()); final DocPropagationSettings defaultSettings = new DocPropagationSettings(); - defaultSettings.setDocColumnPropagation(true); + // TODO: Enable by default. Currently the automation trusts the settings aspect, which does not have this. + defaultSettings.setDocColumnPropagation(false); return globalSettings != null && globalSettings.hasDocPropagation() ? mapDocPropagationSettings(globalSettings.getDocPropagation()) : defaultSettings; diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index e887c29cc4165..eeb51d7fe8f46 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -65,7 +65,7 @@ export default function PropagationDetails({ sourceDetail }: Props) { originEntity || viaEntity ? ( - This description was automatically propagated from an upstream column. Learn more + This description was automatically propagated from an upstream column. Learn more diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx index 24bcd17ca7f9c..e3948349546ef 100644 --- a/datahub-web-react/src/app/settings/SettingsPage.tsx +++ b/datahub-web-react/src/app/settings/SettingsPage.tsx @@ -121,7 +121,7 @@ export const SettingsPage = () => { const showViews = isViewsEnabled || false; const showOwnershipTypes = me && me?.platformPrivileges?.manageOwnershipTypes; const showHomePagePosts = me && me?.platformPrivileges?.manageGlobalAnnouncements && !readOnlyModeEnabled; - const showFeatures = true; // TODO: Add feature flag for this + const showFeatures = me?.platformPrivileges?.manageIngestion; // TODO: Add feature flag for this return ( diff --git a/datahub-web-react/src/app/settings/features/Feature.tsx b/datahub-web-react/src/app/settings/features/Feature.tsx index 3bb1411145abe..873aab9404177 100644 --- a/datahub-web-react/src/app/settings/features/Feature.tsx +++ b/datahub-web-react/src/app/settings/features/Feature.tsx @@ -149,11 +149,13 @@ export const Feature = ({ key, title, description, settings, options, isNew, lea {option.description} - (option.onChange ? option.onChange(checked) : null)} - disabled={!option.isAvailable} - /> + + (option.onChange ? option.onChange(checked) : null)} + disabled={!option.isAvailable || option.isDisabled} + /> + {index !== options.length - 1 && } diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx index 01fdb0d42ffeb..5895c8f2d2a69 100644 --- a/datahub-web-react/src/app/settings/features/Features.tsx +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -80,7 +80,8 @@ export const Features = () => { description: 'Propagate new documentation from upstream to downstream assets based on data lineage relationships.', isAvailable: true, - checked: false, + isDisabled: true, + disabledMessage: 'Coming soon!' }, ], isNew: true, diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index d435f00902d77..058254c18a189 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -94,6 +94,23 @@ module.exports = { }, ], }, + { + label: "Automations", + type: "category", + items: [ + { + label: "Documentation Propagation", + type: "doc", + id: "docs/managed-datahub/automation/docs-propagation", + }, + { + label: "Snowflake Tag Sync", + type: "doc", + id: "docs/managed-datahub/automation/snowflake-tag-propagation", + className: "saasOnly", + }, + ], + }, { label: "Business Attributes", type: "doc", diff --git a/docs/automation/docs-propagation.md b/docs/automation/docs-propagation.md new file mode 100644 index 0000000000000..a637afcde4dca --- /dev/null +++ b/docs/automation/docs-propagation.md @@ -0,0 +1,128 @@ +# Documentation Propagation Automation + +## Introduction + +Documentation Propagation is an automation automatically propagates column and asset (coming soon) descriptions based on downstream column-level lineage and sibling relationships. +It simplifies metadata management by ensuring consistency and reducing the manual effort required for documenting data assets to aid +in Data Governance & Compliance along with Data Discovery. + +This feature is enabled by default in Open Source DataHub. + +## Capabilities + +### Open Source +- **Column-Level Docs Propagation**: Automatically propagate documentation to downstream columns and sibling columns that are derived or dependent on the source column. +- **(Coming Soon) Asset-Level Docs Propagation**: Propagate descriptions to sibling assets. + +### DataHub Cloud (Acryl) +- Includes all the features of Open Source. +- **Propagation Rollback (Undo)**: Offers the ability to undo any propagation changes, providing a safety net against accidental updates. +- **Historical Backfilling**: Automatically backfills historical data for newly documented columns to maintain consistency across time. + +### Comparison of Features + +| Feature | Open Source | DataHub Cloud | +|---------------------------------|-------------|---------------| +| Column-Level Docs Propagation | ✔️ | ✔️ | +| Asset-Level Docs Propagation | ✔️ | ✔️ | +| Downstream Lineage + Siblings | ✔️ | ✔️ | +| Propagation Rollback (Undo) | ❌ | ✔️ | +| Historical Backfilling | ❌ | ✔️ | + +## Enabling Documentation Propagation + +### In Open Source + +Notice that the user must have the `Manage Ingestion` permission to view and enable the feature. + +1. **Navigate to Settings**: Click on the 'Settings' gear in top navigation bar. + +

+ +

+ +2. **Navigate to Features**: Click on the 'Features' tab in the left-hand navigation bar. + +

+ +

+ +3**Enable Documentation Propagation**: Locate the 'Documentation Propagation' section and toggle the feature to enable it for column-level and asset-level propagation. +Currently, Column Level propagation is supported, with asset level propagation coming soon. + +

+ +

+ + +### In DataHub Cloud + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create An Automation**: Click on 'Create' and select 'Column Documentation Propagation'. + +

+ +

+ +3. **Configure Automation**: Fill in the required fields, such as the name, description, and category. Finally, click 'Save and Run' to start the automation + +

+ +

+ +## Propagating for Existing Assets (DataHub Cloud Only) + +In DataHub Cloud, you can back-fill historical data for existing assets to ensure that all existing column descriptions are propagated to downstreams +when you start the automation. Note that it may take some time to complete the initial back-filling process, depending on the number of assets and the complexity of your lineage. + +To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "more" menu: + +

+ +

+ +and then click "Initialize". + +

+ +

+ +This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating +descriptions going forward, you can skip this step. + +## Rolling Back Propagated Descriptions (DataHub Cloud Only) + +In DataHub Cloud, you can rollback all descriptions that have been propagated historically. + +This feature allows you to "clean up" or "undo" any accidental propagation that may have occurred automatically, in the case +that you no longer want propagated descriptions to be visible. + +To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu + +

+ +

+ +and then click "Rollback". + +

+ +

+ +This one-time step will remove all propagated tags and glossary terms from Snowflake. To simply stop propagating new tags, you can disable the automation. + +## Viewing Propagated Descriptions + +Once the automation is enabled, you'll be able to recognize propagated descriptions as those with the thunderbolt icon next to them: + +The tooltip will provide additional information, including where the description originated and any intermediate hops that were +used to propagate the description. + +

+ +

\ No newline at end of file diff --git a/docs/automation/snowflake-tag-propagation.md b/docs/automation/snowflake-tag-propagation.md new file mode 100644 index 0000000000000..bdc80376dfb48 --- /dev/null +++ b/docs/automation/snowflake-tag-propagation.md @@ -0,0 +1,88 @@ + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Snowflake Tag Propagation Automation + + + +## Introduction + +Snowflake Tag Propagation is an automation that allows you to sync DataHub Glossary Terms and Tags on +both columns and tables back to Snowflake. This automation is available in DataHub Cloud (Acryl) only. + +## Capabilities + +- Automatically Add DataHub Glossary Terms to Snowflake Tables and Columns +- Automatically Add DataHub Tags to Snowflake Tables and Columns +- Automatically Remove DataHub Glossary Terms and Tags from Snowflake Tables and Columns when they are removed in DataHub + +## Enabling Snowflake Tag Sync + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create An Automation**: Click on 'Create' and select 'Snowflake Tag Propagation'. + +

+ +

+ +3. **Configure Automation**: Fill in the required fields to connect to Snowflake, along with the name, description, and category. +Note that you can limit propagation based on specific Tags and Glossary Terms. If none are selected, then ALL Tags or Glossary Terms will be automatically +propagated to Snowflake tables and columns. Finally, click 'Save and Run' to start the automation + +

+ +

+ +## Propagating for Existing Assets + +You can back-fill historical data for existing assets to ensure that all existing column and table Tags and Glossary Terms are propagated to Snowflake. +Note that it may take some time to complete the initial back-filling process, depending on the number of Snowflake assets you have. + +To do so, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu + +

+ +

+ +and then click "Initialize". + +

+ +

+ +This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating +descriptions going forward, you can skip this step. + +## Rolling Back Propagated Tags + +You can rollback all tags and glossary terms that have been propagated historically. + +This feature allows you to "clean up" or "undo" any accidental propagation that may have occurred automatically, in the case +that you no longer want propagated descriptions to be visible. + +To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu + +

+ +

+ +and then click "Rollback". + +

+ +

+ +This one-time step will remove all propagated tags and glossary terms from Snowflake. To simply stop propagating new tags, you can disable the automation. + +## Viewing Propagated Tags + +You can view propagated Tags (and corresponding DataHub URNs) inside the Snowflake UI to confirm the automation is working as expected. + +

+ +

diff --git a/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl index 8d4121b767dc3..6c6f4d0036ce0 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl @@ -12,16 +12,17 @@ record GlobalSettingsInfo { * SSO integrations between DataHub and identity providers */ sso: optional SsoSettings + /** * Settings related to the Views Feature */ views: optional GlobalViewsSettings + /** * Settings related to the documentation propagation feature */ - docPropagation: DocPropagationFeatureSettings = { + docPropagation: optional DocPropagationFeatureSettings = { "enabled": true "columnPropagationEnabled": true } - } \ No newline at end of file diff --git a/metadata-service/war/src/main/resources/boot/global_settings.json b/metadata-service/war/src/main/resources/boot/global_settings.json index 129783afd6df4..35145b85202a7 100644 --- a/metadata-service/war/src/main/resources/boot/global_settings.json +++ b/metadata-service/war/src/main/resources/boot/global_settings.json @@ -1,4 +1,8 @@ { "views": { + }, + "docPropagation": { + "enabled": true, + "columnPropagationEnabled": true } } \ No newline at end of file From 54bdf6bb06a00ddfe3eb82a7dbad35531c8689b2 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 9 Aug 2024 17:03:43 -0700 Subject: [PATCH 16/18] Fint lint --- .../shared/propagation/PropagationDetails.tsx | 26 ++++++++++++------- .../propagation/PropagationEntityLink.tsx | 8 +++--- .../src/app/lineage/LineageExplorer.tsx | 4 ++- .../src/app/settings/features/Features.tsx | 4 +-- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index eeb51d7fe8f46..646f47134938c 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -15,19 +15,19 @@ const PopoverTitle = styled.div` font-weight: bold; font-size: 14px; padding: 6px 0px; - color: #EEECFA; + color: #eeecfa; `; const PopoverDescription = styled.div` max-width: 340px; font-size: 14px; - color: #EEECFA; + color: #eeecfa; display: inline; padding: 0px 0px 8px 0px; `; const PopoverAttributes = styled.div` - display: flex; + display: flex; `; const PopoverAttribute = styled.div` @@ -37,7 +37,7 @@ const PopoverAttribute = styled.div` const PopoverAttributeTitle = styled.div` font-size: 14px; - color: #EEECFA; + color: #eeecfa; font-weight: bold; margin: 8px 0px; overflow: hidden; @@ -65,18 +65,26 @@ export default function PropagationDetails({ sourceDetail }: Props) { originEntity || viaEntity ? ( - This description was automatically propagated from an upstream column. Learn more - + This description was automatically propagated from an upstream column.{' '} + + Learn more + {originEntity && originEntity.urn !== viaEntity?.urn && ( - Origin + Origin + )} {viaEntity && ( - Via + Via + )} @@ -85,7 +93,7 @@ export default function PropagationDetails({ sourceDetail }: Props) { return ( diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx index 6ccd54b5bc769..8c1285dd5808b 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationEntityLink.tsx @@ -47,10 +47,10 @@ export default function PropagationEntityLink({ entity }: Props) { return ( <> - - - {entityDisplayName} - + + + {entityDisplayName} + ); } diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx index 0d435b959a186..ce0c4bb8f122d 100644 --- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx @@ -221,7 +221,9 @@ export default function LineageExplorer({ urn, type }: Props) { Close {selectedEntity.type !== EntityType.Restricted && ( - )} diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx index 5895c8f2d2a69..d0145bcb9254b 100644 --- a/datahub-web-react/src/app/settings/features/Features.tsx +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -81,11 +81,11 @@ export const Features = () => { 'Propagate new documentation from upstream to downstream assets based on data lineage relationships.', isAvailable: true, isDisabled: true, - disabledMessage: 'Coming soon!' + disabledMessage: 'Coming soon!', }, ], isNew: true, - learnMoreLink: 'https://datahubproject.io/docs/automations/doc-propagation', + learnMoreLink: 'https://datahubproject.io/docs/automations/docs-propagation', }, ]; From 298e9e47002add3f20c89d3d696e86c16c81a4fd Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 9 Aug 2024 17:19:46 -0700 Subject: [PATCH 17/18] Fixing lint --- datahub-web-react/src/app/settings/features/Feature.tsx | 2 ++ datahub-web-react/src/app/settings/features/Features.tsx | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/datahub-web-react/src/app/settings/features/Feature.tsx b/datahub-web-react/src/app/settings/features/Feature.tsx index 873aab9404177..13453cf8f7325 100644 --- a/datahub-web-react/src/app/settings/features/Feature.tsx +++ b/datahub-web-react/src/app/settings/features/Feature.tsx @@ -104,6 +104,8 @@ export interface FeatureType { title: string; description: string; isAvailable: boolean; + isDisabled: boolean; + disabledMessage?: string; checked: boolean; onChange?: (checked: boolean) => void; }>; diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx index d0145bcb9254b..1d0a0bb469cf8 100644 --- a/datahub-web-react/src/app/settings/features/Features.tsx +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -73,12 +73,16 @@ export const Features = () => { setIsColPropagateChecked(checked); updateDocPropagation(checked); }, + isDisabled: false, + disabledMessage: undefined, }, { key: uuidv4(), title: 'Asset Level Propagation', description: 'Propagate new documentation from upstream to downstream assets based on data lineage relationships.', + checked: false, + onChange: (_: boolean) => null, isAvailable: true, isDisabled: true, disabledMessage: 'Coming soon!', From edbc0dbc8fee5ca28b1dc03ac95d569994eb5783 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Mon, 19 Aug 2024 17:22:45 -0700 Subject: [PATCH 18/18] Adding fixes --- .../docPropagation/DocPropagationSettingsResolver.java | 3 ++- .../graphql/types/chart/mappers/InputFieldsMapper.java | 10 +++++++--- docs-website/sidebars.js | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java index 389a9263e7cfa..0641d6aca6370 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java @@ -33,7 +33,8 @@ public CompletableFuture get(final DataFetchingEnvironme final GlobalSettingsInfo globalSettings = _settingsService.getGlobalSettings(context.getOperationContext()); final DocPropagationSettings defaultSettings = new DocPropagationSettings(); - // TODO: Enable by default. Currently the automation trusts the settings aspect, which does not have this. + // TODO: Enable by default. Currently the automation trusts the settings aspect, which + // does not have this. defaultSettings.setDocColumnPropagation(false); return globalSettings != null && globalSettings.hasDocPropagation() ? mapDocPropagationSettings(globalSettings.getDocPropagation()) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java index a4e40750f0d65..269fb7d4ddf79 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java @@ -11,7 +11,6 @@ import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; - @Slf4j public class InputFieldsMapper { @@ -40,9 +39,14 @@ public com.linkedin.datahub.graphql.generated.InputFields apply( if (field.hasSchemaFieldUrn()) { fieldResult.setSchemaFieldUrn(field.getSchemaFieldUrn().toString()); try { - parentUrn = Urn.createFromString(field.getSchemaFieldUrn().getEntityKey().get(0)); + parentUrn = + Urn.createFromString(field.getSchemaFieldUrn().getEntityKey().get(0)); } catch (URISyntaxException e) { - log.error("Field urn resolution: failed to extract parentUrn successfully from {}. Falling back to {}", field.getSchemaFieldUrn(), entityUrn, e); + log.error( + "Field urn resolution: failed to extract parentUrn successfully from {}. Falling back to {}", + field.getSchemaFieldUrn(), + entityUrn, + e); } } if (field.hasSchemaField()) { diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 058254c18a189..2efdebb6bd738 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -101,12 +101,12 @@ module.exports = { { label: "Documentation Propagation", type: "doc", - id: "docs/managed-datahub/automation/docs-propagation", + id: "docs/automation/docs-propagation", }, { label: "Snowflake Tag Sync", type: "doc", - id: "docs/managed-datahub/automation/snowflake-tag-propagation", + id: "docs/automation/snowflake-tag-propagation", className: "saasOnly", }, ],