diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7d94ffa7748a9..6ee40cc24ed98 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -21,6 +21,14 @@ concurrency: jobs: build: + strategy: + matrix: + command: + [ + "./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :datahub-frontend:build -x :datahub-web-react:build --parallel", + "./gradlew :datahub-frontend:build :datahub-web-react:build --parallel", + "./gradlew :metadata-ingestion-modules:airflow-plugin:build --parallel" + ] runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -37,7 +45,7 @@ jobs: python-version: "3.7" - name: Gradle build (and test) run: | - ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test + ${{ matrix.command }} - uses: actions/upload-artifact@v3 if: always() with: diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index 79b714791ff49..6641851029383 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -4,7 +4,7 @@ on: branches: - master paths: - - ".github/**" + - ".github/workflows/metadata-ingestion.yml" - "metadata-ingestion/**" - "metadata-models/**" pull_request: diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java index afd6bf06afb94..fab17f9215d4a 100644 --- a/datahub-frontend/app/client/KafkaTrackingProducer.java +++ b/datahub-frontend/app/client/KafkaTrackingProducer.java @@ -94,6 +94,7 @@ private static KafkaProducer createKafkaProducer(Config config) { setConfig(config, props, SaslConfigs.SASL_JAAS_CONFIG, "analytics.kafka.sasl.jaas.config"); setConfig(config, props, SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "analytics.kafka.sasl.kerberos.service.name"); setConfig(config, props, SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, "analytics.kafka.sasl.login.callback.handler.class"); + setConfig(config, props, SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, "analytics.kafka.sasl.client.callback.handler.class"); } } diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index ff836e2720572..1a83f44981e4d 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -210,6 +210,7 @@ analytics.kafka.sasl.mechanism = ${?KAFKA_PROPERTIES_SASL_MECHANISM} analytics.kafka.sasl.jaas.config = ${?KAFKA_PROPERTIES_SASL_JAAS_CONFIG} analytics.kafka.sasl.kerberos.service.name = ${?KAFKA_PROPERTIES_SASL_KERBEROS_SERVICE_NAME} analytics.kafka.sasl.login.callback.handler.class = ${?KAFKA_PROPERTIES_SASL_LOGIN_CALLBACK_HANDLER_CLASS} +analytics.kafka.sasl.client.callback.handler.class = ${?KAFKA_PROPERTIES_SASL_CLIENT_CALLBACK_HANDLER_CLASS} # Required Elastic Client Configuration analytics.elastic.host = ${?ELASTIC_CLIENT_HOST} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 78fb381b02b11..d28b62d3e15ec 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -111,6 +111,7 @@ import com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver; import com.linkedin.datahub.graphql.resolvers.domain.SetDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.UnsetDomainResolver; +import com.linkedin.datahub.graphql.resolvers.embed.UpdateEmbedResolver; import com.linkedin.datahub.graphql.resolvers.entity.EntityExistsResolver; import com.linkedin.datahub.graphql.resolvers.entity.EntityPrivilegesResolver; import com.linkedin.datahub.graphql.resolvers.glossary.AddRelatedTermsResolver; @@ -852,6 +853,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("updateGlobalViewsSettings", new UpdateGlobalViewsSettingsResolver(this.settingsService)) .dataFetcher("updateCorpUserViewsSettings", new UpdateCorpUserViewsSettingsResolver(this.settingsService)) .dataFetcher("updateLineage", new UpdateLineageResolver(this.entityService, this.lineageService)) + .dataFetcher("updateEmbed", new UpdateEmbedResolver(this.entityService)) ); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 0272708c6461b..22ca0b05f0760 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -12,6 +12,7 @@ import com.linkedin.datahub.graphql.generated.ManagedIngestionConfig; import com.linkedin.datahub.graphql.generated.PoliciesConfig; import com.linkedin.datahub.graphql.generated.Privilege; +import com.linkedin.datahub.graphql.generated.QueriesTabConfig; import com.linkedin.datahub.graphql.generated.ResourcePrivileges; import com.linkedin.datahub.graphql.generated.TelemetryConfig; import com.linkedin.datahub.graphql.generated.TestsConfig; @@ -122,6 +123,11 @@ public CompletableFuture get(final DataFetchingEnvironment environmen visualConfig.setLogoUrl(_visualConfiguration.getAssets().getLogoUrl()); visualConfig.setFaviconUrl(_visualConfiguration.getAssets().getFaviconUrl()); } + if (_visualConfiguration != null && _visualConfiguration.getQueriesTab() != null) { + QueriesTabConfig queriesTabConfig = new QueriesTabConfig(); + queriesTabConfig.setQueriesTabResultSize(_visualConfiguration.getQueriesTab().getQueriesTabResultSize()); + visualConfig.setQueriesTab(queriesTabConfig); + } appConfig.setVisualConfig(visualConfig); final TelemetryConfig telemetryConfig = new TelemetryConfig(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java new file mode 100644 index 0000000000000..3d4a55b0cdbb6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java @@ -0,0 +1,104 @@ +package com.linkedin.datahub.graphql.resolvers.embed; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Embed; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.UpdateEmbedInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.EmbedUtils; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + + +/** + * Resolver used for updating the embed render URL for an asset. + */ +@Slf4j +@RequiredArgsConstructor +public class UpdateEmbedResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final UpdateEmbedInput input = bindArgument(environment.getArgument("input"), UpdateEmbedInput.class); + final Urn entityUrn = UrnUtils.getUrn(input.getUrn()); + + return CompletableFuture.supplyAsync(() -> { + + if (!EmbedUtils.isAuthorizedToUpdateEmbedForEntity(entityUrn, environment.getContext())) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + validateUpdateEmbedInput( + input, + _entityService + ); + try { + final Embed embed = (Embed) getAspectFromEntity( + entityUrn.toString(), + Constants.EMBED_ASPECT_NAME, + _entityService, + new Embed()); + + updateEmbed(embed, input); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(entityUrn); + proposal.setEntityType(entityUrn.getEntityType()); + proposal.setAspectName(Constants.EMBED_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(embed)); + proposal.setChangeType(ChangeType.UPSERT); + _entityService.ingestProposal( + proposal, + new AuditStamp().setActor(UrnUtils.getUrn(context.getActorUrn())).setTime(System.currentTimeMillis()), + false + ); + return true; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to update Embed for to resource with entity urn %s", entityUrn), e); + } + }); + } + + /** + * Validates an instance of {@link UpdateEmbedInput}, and throws an {@link IllegalArgumentException} if the input + * is not valid. + * + * For an input to be valid, the target URN must exist. + * + * @param input the input to validate + * @param entityService an instance of {@link EntityService} used to validate the input. + */ + private static void validateUpdateEmbedInput(@Nonnull final UpdateEmbedInput input, @Nonnull final EntityService entityService) { + if (!entityService.exists(UrnUtils.getUrn(input.getUrn()))) { + throw new IllegalArgumentException( + String.format("Failed to update embed for entity with urn %s. Entity does not exist!", input.getUrn())); + } + } + + /** + * Applies an instance of {@link UpdateEmbedInput} to a base instance of {@link Embed}. + * @param embed an embed to update + * @param input the updates to apply + */ + private static void updateEmbed(@Nonnull final Embed embed, @Nonnull final UpdateEmbedInput input) { + embed.setRenderUrl(input.getRenderUrl(), SetMode.IGNORE_NULL); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java index 046a28fb9fb3e..4f0f98c117da3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java @@ -10,6 +10,7 @@ import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityPrivileges; import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.EmbedUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; @@ -110,18 +111,21 @@ private boolean canEditEntityLineage(Urn urn, QueryContext context) { private EntityPrivileges getDatasetPrivileges(Urn urn, QueryContext context) { final EntityPrivileges result = new EntityPrivileges(); result.setCanEditLineage(canEditEntityLineage(urn, context)); + result.setCanEditEmbed(EmbedUtils.isAuthorizedToUpdateEmbedForEntity(urn, context)); return result; } private EntityPrivileges getChartPrivileges(Urn urn, QueryContext context) { final EntityPrivileges result = new EntityPrivileges(); result.setCanEditLineage(canEditEntityLineage(urn, context)); + result.setCanEditEmbed(EmbedUtils.isAuthorizedToUpdateEmbedForEntity(urn, context)); return result; } private EntityPrivileges getDashboardPrivileges(Urn urn, QueryContext context) { final EntityPrivileges result = new EntityPrivileges(); result.setCanEditLineage(canEditEntityLineage(urn, context)); + result.setCanEditEmbed(EmbedUtils.isAuthorizedToUpdateEmbedForEntity(urn, context)); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java new file mode 100644 index 0000000000000..635419208c8fb --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java @@ -0,0 +1,36 @@ +package com.linkedin.datahub.graphql.resolvers.mutate.util; + +import com.google.common.collect.ImmutableList; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.metadata.authorization.PoliciesConfig; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class EmbedUtils { + private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() + )); + + private EmbedUtils() { } + + public static boolean isAuthorizedToUpdateEmbedForEntity(@Nonnull final Urn entityUrn, @Nonnull final QueryContext context) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of( + ALL_PRIVILEGES_GROUP, + new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.EDIT_ENTITY_EMBED_PRIVILEGE.getType())) + )); + + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + entityUrn.getEntityType(), + entityUrn.toString(), + orPrivilegeGroups); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index b797e8ca97eb3..3768faee29860 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -72,7 +72,8 @@ public class ChartType implements SearchableEntityType, Browsable DOMAINS_ASPECT_NAME, DEPRECATION_ASPECT_NAME, DATA_PLATFORM_INSTANCE_ASPECT_NAME, - INPUT_FIELDS_ASPECT_NAME + INPUT_FIELDS_ASPECT_NAME, + EMBED_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("access", "queryType", "tool", "type"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java index 0d75832d2b1a8..6aebf3f9bb7ef 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java @@ -3,6 +3,7 @@ import com.linkedin.chart.EditableChartProperties; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; +import com.linkedin.common.Embed; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTerms; import com.linkedin.common.InputFields; @@ -26,6 +27,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; +import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; @@ -89,7 +91,8 @@ public Chart apply(@Nonnull final EntityResponse entityResponse) { dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); mappingHelper.mapToResult(INPUT_FIELDS_ASPECT_NAME, (chart, dataMap) -> chart.setInputFields(InputFieldsMapper.map(new InputFields(dataMap), entityUrn))); - + mappingHelper.mapToResult(EMBED_ASPECT_NAME, (chart, dataMap) -> + chart.setEmbed(EmbedMapper.map(new Embed(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/EmbedMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/EmbedMapper.java new file mode 100644 index 0000000000000..478d256df66a4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/EmbedMapper.java @@ -0,0 +1,22 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.datahub.graphql.generated.Embed; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; + +import javax.annotation.Nonnull; + +public class EmbedMapper implements ModelMapper { + + public static final EmbedMapper INSTANCE = new EmbedMapper(); + + public static Embed map(@Nonnull final com.linkedin.common.Embed metadata) { + return INSTANCE.apply(metadata); + } + + @Override + public Embed apply(@Nonnull final com.linkedin.common.Embed input) { + final Embed result = new Embed(); + result.setRenderUrl(input.getRenderUrl()); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index a84499d32283d..fede40ca3d7db 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -74,7 +74,8 @@ public class DashboardType implements SearchableEntityType, B DEPRECATION_ASPECT_NAME, DATA_PLATFORM_INSTANCE_ASPECT_NAME, INPUT_FIELDS_ASPECT_NAME, - SUB_TYPES_ASPECT_NAME + SUB_TYPES_ASPECT_NAME, + EMBED_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("access", "tool"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java index 1fe9861fe18ef..709a1678eefda 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java @@ -2,6 +2,7 @@ import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; +import com.linkedin.common.Embed; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTerms; import com.linkedin.common.InputFields; @@ -25,6 +26,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; +import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; @@ -88,7 +90,8 @@ public Dashboard apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(INPUT_FIELDS_ASPECT_NAME, (dashboard, dataMap) -> dashboard.setInputFields(InputFieldsMapper.map(new InputFields(dataMap), entityUrn))); mappingHelper.mapToResult(SUB_TYPES_ASPECT_NAME, this::mapSubTypes); - + mappingHelper.mapToResult(EMBED_ASPECT_NAME, (dashboard, dataMap) -> + dashboard.setEmbed(EmbedMapper.map(new Embed(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index e4898256b56c7..8dd5ba8675242 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -80,7 +80,8 @@ public class DatasetType implements SearchableEntityType, Brows DOMAINS_ASPECT_NAME, SCHEMA_METADATA_ASPECT_NAME, DATA_PLATFORM_INSTANCE_ASPECT_NAME, - SIBLINGS_ASPECT_NAME + SIBLINGS_ASPECT_NAME, + EMBED_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 4a36296a22e5d..781f9ed29f054 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -2,6 +2,7 @@ import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; +import com.linkedin.common.Embed; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTerms; import com.linkedin.common.InstitutionalMemory; @@ -18,6 +19,7 @@ import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; +import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.SiblingsMapper; @@ -101,7 +103,8 @@ public Dataset apply(@Nonnull final EntityResponse entityResponse) { dataset.setSiblings(SiblingsMapper.map(new Siblings(dataMap)))); mappingHelper.mapToResult(UPSTREAM_LINEAGE_ASPECT_NAME, (dataset, dataMap) -> dataset.setFineGrainedLineages(UpstreamLineagesMapper.map(new UpstreamLineage(dataMap)))); - + mappingHelper.mapToResult(EMBED_ASPECT_NAME, (dataset, dataMap) -> + dataset.setEmbed(EmbedMapper.map(new Embed(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 333c932c0d1ad..4abc99eba264d 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -196,6 +196,21 @@ type VisualConfig { Custom favicon url for the homepage & top banner """ faviconUrl: String + + """ + Configuration for the queries tab + """ + queriesTab: QueriesTabConfig +} + +""" +Configuration for the queries tab +""" +type QueriesTabConfig { + """ + Number of queries to show in the queries tab + """ + queriesTabResultSize: Int } """ diff --git a/datahub-graphql-core/src/main/resources/auth.graphql b/datahub-graphql-core/src/main/resources/auth.graphql index 2873f2ba81aca..4ed24c854bc1a 100644 --- a/datahub-graphql-core/src/main/resources/auth.graphql +++ b/datahub-graphql-core/src/main/resources/auth.graphql @@ -258,4 +258,9 @@ type EntityPrivileges { Whether or not a user can create or delete lineage edges for an entity. """ canEditLineage: Boolean + + """ + Whether or not a user update the embed information + """ + canEditEmbed: Boolean } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 3d76393219fe5..faa38d223ebfb 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -576,6 +576,11 @@ type Mutation { Update lineage for an entity """ updateLineage(input: UpdateLineageInput!): Boolean + + """ + Update the Embed information for a Dataset, Dashboard, or Chart. + """ + updateEmbed(input: UpdateEmbedInput!): Boolean } """ @@ -1086,6 +1091,11 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity { """ status: Status + """ + Embed information about the Dataset + """ + embed: Embed + """ Tags used for searching dataset """ @@ -4418,6 +4428,11 @@ type Dashboard implements EntityWithRelationships & Entity & BrowsableEntity { """ status: Status + """ + Embed information about the Dashboard + """ + embed: Embed + """ The deprecation status of the dashboard """ @@ -4721,6 +4736,11 @@ type Chart implements EntityWithRelationships & Entity & BrowsableEntity { """ deprecation: Deprecation + """ + Embed information about the Chart + """ + embed: Embed + """ The tags associated with the chart """ @@ -10017,4 +10037,29 @@ input UpdateCorpUserViewsSettingsInput { If not provided, any existing default view will be removed. """ defaultView: String +} + +""" +Information required to render an embedded version of an asset +""" +type Embed { + """ + A URL which can be rendered inside of an iframe. + """ + renderUrl: String +} + +""" +Input required to set or clear information related to rendering a Data Asset inside of DataHub. +""" +input UpdateEmbedInput { + """ + The URN associated with the Data Asset to update. Only dataset, dashboard, and chart urns are currently supported. + """ + urn: String! + + """ + Set or clear a URL used to render an embedded asset. + """ + renderUrl: String } \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java new file mode 100644 index 0000000000000..94dc742d77831 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java @@ -0,0 +1,198 @@ +package com.linkedin.datahub.graphql.resolvers.embed; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Embed; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.UpdateEmbedInput; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; +import java.util.HashSet; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class UpdateEmbedResolverTest { + + private static final String TEST_ENTITY_URN = "urn:li:dashboard:(looker,1)"; + private static final String TEST_RENDER_URL = "https://www.google.com"; + private static final UpdateEmbedInput TEST_EMBED_INPUT = new UpdateEmbedInput( + TEST_ENTITY_URN, + TEST_RENDER_URL + ); + private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); + + @Test + public void testGetSuccessNoExistingEmbed() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), + Mockito.eq(Constants.EMBED_ASPECT_NAME), + Mockito.eq(0L))).thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + UpdateEmbedResolver resolver = new UpdateEmbedResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_EMBED_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + resolver.get(mockEnv).get(); + + final Embed newEmbed = new Embed().setRenderUrl(TEST_RENDER_URL); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); + proposal.setEntityType(Constants.DASHBOARD_ENTITY_NAME); + proposal.setAspectName(Constants.EMBED_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(newEmbed)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + } + + @Test + public void testGetSuccessExistingEmbed() throws Exception { + Embed originalEmbed = new Embed().setRenderUrl("https://otherurl.com"); + + // Create resolver + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), + Mockito.eq(Constants.EMBED_ASPECT_NAME), + Mockito.eq(0L))).thenReturn(originalEmbed); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + UpdateEmbedResolver resolver = new UpdateEmbedResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_EMBED_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + resolver.get(mockEnv).get(); + + final Embed newEmbed = new Embed().setRenderUrl(TEST_RENDER_URL); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); + proposal.setEntityType(Constants.DASHBOARD_ENTITY_NAME); + proposal.setAspectName(Constants.EMBED_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(newEmbed)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + } + + @Test + public void testGetFailureEntityDoesNotExist() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.batchGetV2( + Mockito.eq(Constants.DASHBOARD_ENTITY_NAME), + Mockito.eq(new HashSet<>(ImmutableSet.of(Urn.createFromString(TEST_ENTITY_URN)))), + Mockito.eq(ImmutableSet.of(Constants.EMBED_ASPECT_NAME)), + Mockito.any(Authentication.class))) + .thenReturn(ImmutableMap.of(Urn.createFromString(TEST_ENTITY_URN), + new EntityResponse() + .setEntityName(Constants.DASHBOARD_ENTITY_NAME) + .setUrn(Urn.createFromString(TEST_ENTITY_URN)) + .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); + + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(false); + + UpdateEmbedResolver resolver = new UpdateEmbedResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_EMBED_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + );; + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityService mockService = Mockito.mock(EntityService.class); + UpdateEmbedResolver resolver = new UpdateEmbedResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_EMBED_INPUT); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + ); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.doThrow(RemoteInvocationException.class).when(mockClient).ingestProposal( + Mockito.any(), + Mockito.any(Authentication.class)); + UpdateEmbedResolver resolver = new UpdateEmbedResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_EMBED_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/EmbedMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/EmbedMapperTest.java new file mode 100644 index 0000000000000..54b341fc1865a --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/EmbedMapperTest.java @@ -0,0 +1,14 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.datahub.graphql.generated.Embed; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class EmbedMapperTest { + @Test + public void testEmbedMapper() throws Exception { + final String renderUrl = "https://www.google.com"; + final Embed result = EmbedMapper.map(new com.linkedin.common.Embed().setRenderUrl(renderUrl)); + Assert.assertEquals(result.getRenderUrl(), renderUrl); + } +} diff --git a/datahub-web-react/README.md b/datahub-web-react/README.md index 90b0e375af4c9..6c91b169af858 100644 --- a/datahub-web-react/README.md +++ b/datahub-web-react/README.md @@ -53,15 +53,11 @@ Optionally you could also start the app with the mock server without running the ### Functional testing -Automated functional testing is powered by Cypress and MirageJS. When running the web server with Cypress the port is set to 3010 so that the usual web server running on port 3000 used for development can be started without interruptions. +In order to start a server and run frontend unit tests using react-testing-framework, run: -#### During development +`yarn test :e2e` -`yarn test:e2e` - -#### CI - -`yarn test:e2e:ci` +There are also more automated tests using Cypress in the `smoke-test` folder of the repository root. #### Troubleshooting `Error: error:0308010C:digital envelope routines::unsupported`: This error message shows up when using Node 17, due to an OpenSSL update related to md5. diff --git a/datahub-web-react/cypress.json b/datahub-web-react/cypress.json deleted file mode 100644 index c3d02c59545f0..0000000000000 --- a/datahub-web-react/cypress.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "baseUrl": "http://localhost:3010", - "video": false -} diff --git a/datahub-web-react/cypress/.eslintrc.js b/datahub-web-react/cypress/.eslintrc.js deleted file mode 100644 index 6cb6d237f5f0a..0000000000000 --- a/datahub-web-react/cypress/.eslintrc.js +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = { - extends: '../.eslintrc.js', - parserOptions: { - project: 'cypress/tsconfig.json', - }, - globals: { - Cypress: true, - }, - rules: { - 'jest/expect-expect': 0, - }, -}; diff --git a/datahub-web-react/cypress/fixtures/example.json b/datahub-web-react/cypress/fixtures/example.json deleted file mode 100644 index 02e4254378e97..0000000000000 --- a/datahub-web-react/cypress/fixtures/example.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "Using fixtures to represent data", - "email": "hello@cypress.io", - "body": "Fixtures are a great way to mock data for responses to routes" -} diff --git a/datahub-web-react/cypress/helper/authHelper.ts b/datahub-web-react/cypress/helper/authHelper.ts deleted file mode 100644 index 4cf7a2a76b6b6..0000000000000 --- a/datahub-web-react/cypress/helper/authHelper.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const login = (username) => { - cy.visit('/'); - cy.get('input#username').type(username); - cy.get('input#password').type(username); - cy.contains('Log in').click(); -}; - -export const logout = (username) => { - cy.get(`a[href="/user/urn:li:corpuser:${username}"]`).children('.anticon.anticon-caret-down').trigger('mouseover'); - cy.get('li#user-profile-menu-logout').click(); -}; diff --git a/datahub-web-react/cypress/integration/Login.spec.ts b/datahub-web-react/cypress/integration/Login.spec.ts deleted file mode 100644 index 43e87634dee39..0000000000000 --- a/datahub-web-react/cypress/integration/Login.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { createLoginUsers } from '../../src/graphql-mock/fixtures/user'; -import { makeServer } from '../../src/graphql-mock/server'; -import { login, logout } from '../helper/authHelper'; - -describe('Login', () => { - let server; - - beforeEach(() => { - server = makeServer('test'); - createLoginUsers(server); - }); - - afterEach(() => { - server.shutdown(); - }); - - describe('given the login page is loaded', () => { - describe('when logging in with incorrect credentials', () => { - it('then the login should fail and the toast notification should be briefly displayed', () => { - login('kafkaa'); - - cy.contains('Failed to log in!').should('be.visible'); - }); - }); - - describe('when logging in with correct credentials', () => { - it('then the home page should be displayed', () => { - login('kafka'); - - cy.contains('Welcome back,').should('be.visible'); - cy.contains('Datasets').should('be.visible'); - cy.contains('Dashboard').should('be.visible'); - cy.contains('Chart').should('be.visible'); - cy.contains('Pipelines').should('be.visible'); - - logout('kafka'); - }); - }); - }); -}); diff --git a/datahub-web-react/cypress/integration/Search.spec.ts b/datahub-web-react/cypress/integration/Search.spec.ts deleted file mode 100644 index 8585a124e7c6a..0000000000000 --- a/datahub-web-react/cypress/integration/Search.spec.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { createLoginUsers } from '../../src/graphql-mock/fixtures/user'; -import { makeServer } from '../../src/graphql-mock/server'; -import { login, logout } from '../helper/authHelper'; - -describe('Search', () => { - let server; - - beforeEach(() => { - server = makeServer('test'); - createLoginUsers(server); - }); - - afterEach(() => { - server.shutdown(); - }); - - describe('given the home page is loaded', () => { - describe('when the user enters a keyword in the search field and results found and the first item is selected from the search result dropdown', () => { - it('then the search result page should be displayed with the Task tab be selected and the selected item be displayed', () => { - login('kafka'); - - cy.get('input[placeholder="Search Datasets, People, & more..."]').type('load'); - - cy.get('div.rc-virtual-list-holder-inner') - .children('div.ant-select-item.ant-select-item-option.ant-select-item-option-grouped') - .contains('load_all_') - .click(); - - cy.get('.ant-tabs-tab.ant-tabs-tab-active').contains('Task').should('be.visible'); - cy.contains('load_all_').should('be.visible'); - - logout('kafka'); - }); - }); - }); -}); diff --git a/datahub-web-react/cypress/plugins/index.js b/datahub-web-react/cypress/plugins/index.js deleted file mode 100644 index de2161b56d937..0000000000000 --- a/datahub-web-react/cypress/plugins/index.js +++ /dev/null @@ -1,47 +0,0 @@ -/// -// *********************************************************** -// This example plugins/index.js can be used to load plugins -// -// You can change the location of this file or turn off loading -// the plugins file with the 'pluginsFile' configuration option. -// -// You can read more here: -// https://on.cypress.io/plugins-guide -// *********************************************************** - -// This function is called when a project is opened or re-opened (e.g. due to -// the project's config changing) - -const findWebpack = require('find-webpack'); -const wp = require('@cypress/webpack-preprocessor'); - -/** - * @type {Cypress.PluginConfig} - */ -module.exports = (on, config) => { - // find the Webpack config used by react-scripts - const webpackOptions = findWebpack.getWebpackOptions(); - - if (!webpackOptions) { - throw new Error('Could not find Webpack in this project 😢'); - } - - const cleanOptions = { - reactScripts: true, - }; - - findWebpack.cleanForCypress(cleanOptions, webpackOptions); - - const options = { - webpackOptions, - watchOptions: {}, - }; - - on('file:preprocessor', wp(options)); - - // add other tasks to be registered here - - // IMPORTANT to return the config object - // with the any changed environment variables - return config; -}; diff --git a/datahub-web-react/cypress/support/index.js b/datahub-web-react/cypress/support/index.js deleted file mode 100644 index 725a6cb98a8ad..0000000000000 --- a/datahub-web-react/cypress/support/index.js +++ /dev/null @@ -1,20 +0,0 @@ -/* eslint-disable no-param-reassign */ - -Cypress.on('window:before:load', (win) => { - win.handleFromCypress = (request) => { - return fetch(request.url, { - method: request.method, - headers: request.requestHeaders, - body: request.requestBody, - }) - .then((res) => { - const content = res.headers.get('content-type').includes('application/json') ? res.json() : res.text(); - return new Promise((resolve) => { - content.then((body) => resolve([res.status, res.headers, body])); - }); - }) - .catch((error) => { - console.log('Cypress request proxy error', { error }); - }); - }; -}); diff --git a/datahub-web-react/cypress/tsconfig.json b/datahub-web-react/cypress/tsconfig.json deleted file mode 100644 index 95042c164f2a4..0000000000000 --- a/datahub-web-react/cypress/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "target": "es5", - "lib": ["es5", "dom"], - "types": ["cypress"], - "noEmit": false - }, - "include": ["../node_modules/cypress", "**/*.ts", "support/index.js"] -} diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index a38eaec9b9037..dcdb67db88430 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -5,25 +5,20 @@ "dependencies": { "@analytics/amplitude": "0.0.3", "@analytics/google-analytics": "^0.5.2", - "@analytics/google-tag-manager": "^0.5.0", "@analytics/mixpanel": "^0.3.1", - "@analytics/segment": "^1.1.0", "@ant-design/colors": "^5.0.0", "@ant-design/icons": "^4.3.0", "@apollo/client": "^3.3.19", "@craco/craco": "^6.1.1", - "@cypress/webpack-preprocessor": "5.8.0", "@data-ui/xy-chart": "^0.0.84", "@miragejs/graphql": "^0.1.11", "@monaco-editor/react": "^4.3.1", "@react-hook/window-size": "^3.0.7", "@testing-library/jest-dom": "^5.11.6", "@testing-library/react": "^11.2.2", - "@testing-library/user-event": "^12.6.0", "@tommoor/remove-markdown": "^0.3.2", "@types/diff": "^5.0.0", "@types/dompurify": "^2.3.3", - "@types/jest": "^26.0.19", "@types/js-cookie": "^2.2.6", "@types/node": "^12.19.9", "@types/react": "^17.0.0", @@ -31,11 +26,7 @@ "@types/react-router": "^5.1.8", "@types/react-router-dom": "^5.1.6", "@uiw/react-md-editor": "^3.3.4", - "@visx/drag": "^1.7.0", - "@visx/responsive": "^1.7.0", "@vx/axis": "^0.0.199", - "@vx/gradient": "^0.0.199", - "@vx/grid": "^0.0.199", "@vx/group": "^0.0.199", "@vx/hierarchy": "^0.0.199", "@vx/legend": "^0.0.199", @@ -44,15 +35,10 @@ "@vx/zoom": "^0.0.199", "analytics": "^0.7.5", "antd": "4.17.4", - "apollo-link": "^1.2.14", - "apollo-link-error": "^1.1.13", - "apollo-link-http": "^1.5.17", "color-hash": "^2.0.1", "craco-antd": "^1.19.0", "cronstrue": "^1.122.0", - "cypress": "7.3.0", "d3-scale": "^3.3.0", - "d3-time-format": "^3.0.0", "deepmerge": "^4.2.2", "diff": "^5.0.0", "dompurify": "^2.3.8", @@ -64,11 +50,9 @@ "graphql.macro": "^1.4.2", "history": "^5.0.0", "js-cookie": "^2.2.1", - "lodash.debounce": "^4.0.8", "miragejs": "^0.1.41", "moment-timezone": "^0.5.35", "monaco-editor": "^0.28.1", - "monaco-yaml": "^3.2.1", "query-string": "^6.13.8", "rc-table": "^7.13.1", "react": "^17.0.0", @@ -83,7 +67,6 @@ "react-router-dom": "^5.1.6", "react-scripts": "4.0.3", "react-syntax-highlighter": "^15.4.4", - "react-timezone-select": "^1.1.15", "react-visibility-sensor": "^5.1.1", "reactour": "1.18.7", "sinon": "^11.1.1", @@ -92,7 +75,6 @@ "typescript": "^4.1.3", "uuid": "^8.3.2", "virtualizedtableforantd4": "^1.2.1", - "vx": "^0.0.1", "web-vitals": "^0.2.4", "yamljs": "^0.3.0" }, @@ -104,11 +86,8 @@ "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start", "build": "yarn run generate && CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/", "test": "craco test", - "cy:run:ci": "cypress run", "pretest:e2e:ci": "yarn generate", - "test:e2e:ci": "start-server-and-test start:e2e 3010 cy:run:ci", - "cy:open": "cypress open", - "test:e2e": "start-server-and-test start:e2e 3010 cy:open", + "test:e2e": "start-server-and-test start:e2e 3010", "eject": "react-scripts eject", "generate": "graphql-codegen --config codegen.yml", "lint": "eslint . --ext .ts,.tsx --quiet", @@ -135,8 +114,6 @@ "devDependencies": { "@graphql-codegen/cli": "1.20.0", "@graphql-codegen/near-operation-file-preset": "^1.17.13", - "@graphql-codegen/typescript": "1.20.0", - "@graphql-codegen/typescript-document-nodes": "1.17.9", "@graphql-codegen/typescript-operations": "1.17.13", "@graphql-codegen/typescript-react-apollo": "2.2.1", "@types/graphql": "^14.5.0", diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 35d091993d3af..bbba1347dd2e4 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -155,6 +155,7 @@ export const dataset1 = { uri: 'www.google.com', privileges: { canEditLineage: false, + canEditEmbed: false, }, properties: { name: 'The Great Test Dataset', @@ -235,6 +236,7 @@ export const dataset1 = { deprecation: null, testResults: null, statsSummary: null, + embed: null, }; export const dataset2 = { @@ -253,6 +255,7 @@ export const dataset2 = { }, privileges: { canEditLineage: false, + canEditEmbed: false, }, lastIngested: null, dataPlatformInstance: null, @@ -325,6 +328,7 @@ export const dataset2 = { deprecation: null, testResults: null, statsSummary: null, + embed: null, }; export const dataset3 = { @@ -344,6 +348,7 @@ export const dataset3 = { }, privileges: { canEditLineage: false, + canEditEmbed: false, }, lastIngested: null, dataPlatformInstance: null, @@ -552,6 +557,7 @@ export const dataset3 = { testResults: null, siblings: null, statsSummary: null, + embed: null, } as Dataset; export const dataset4 = { @@ -1223,6 +1229,7 @@ export const dataJob1 = { }, privileges: { canEditLineage: false, + canEditEmbed: false, }, properties: { name: 'DataJobInfoName', @@ -1284,6 +1291,7 @@ export const dataJob2 = { jobId: 'jobId2', privileges: { canEditLineage: false, + canEditEmbed: false, }, ownership: { __typename: 'Ownership', @@ -1353,6 +1361,7 @@ export const dataJob3 = { lastIngested: null, privileges: { canEditLineage: false, + canEditEmbed: false, }, ownership: { __typename: 'Ownership', diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx index 706656d2a9627..507b89f9a6a0a 100644 --- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx +++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx @@ -20,6 +20,7 @@ import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; import { ChartStatsSummarySubHeader } from './profile/stats/ChartStatsSummarySubHeader'; import { InputFieldsTab } from '../shared/tabs/Entity/InputFieldsTab'; import { ChartSnippet } from './ChartSnippet'; +import { EmbedTab } from '../shared/tabs/Embed/EmbedTab'; /** * Definition of the DataHub Chart entity. @@ -91,8 +92,12 @@ export class ChartEntity implements Entity { }, }, { - name: 'Properties', - component: PropertiesTab, + name: 'Preview', + component: EmbedTab, + display: { + visible: (_, chart: GetChartQuery) => !!chart?.chart?.embed?.renderUrl, + enabled: (_, chart: GetChartQuery) => !!chart?.chart?.embed?.renderUrl, + }, }, { name: 'Lineage', @@ -109,6 +114,10 @@ export class ChartEntity implements Entity { enabled: (_, chart: GetChartQuery) => (chart?.chart?.dashboards?.total || 0) > 0, }, }, + { + name: 'Properties', + component: PropertiesTab, + }, ]} sidebarSections={[ { diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx index 0cef664f5b663..3bc21b2cce987 100644 --- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx +++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx @@ -25,6 +25,7 @@ import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import { DashboardStatsSummarySubHeader } from './profile/DashboardStatsSummarySubHeader'; import { ChartSnippet } from '../chart/ChartSnippet'; +import { EmbedTab } from '../shared/tabs/Embed/EmbedTab'; /** * Definition of the DataHub Dashboard entity. @@ -98,8 +99,12 @@ export class DashboardEntity implements Entity { component: DocumentationTab, }, { - name: 'Properties', - component: PropertiesTab, + name: 'Preview', + component: EmbedTab, + display: { + visible: (_, dashboard: GetDashboardQuery) => !!dashboard?.dashboard?.embed?.renderUrl, + enabled: (_, dashboard: GetDashboardQuery) => !!dashboard?.dashboard?.embed?.renderUrl, + }, }, { name: 'Lineage', @@ -108,6 +113,10 @@ export class DashboardEntity implements Entity { defaultDirection: LineageDirection.Upstream, }, }, + { + name: 'Properties', + component: PropertiesTab, + }, { name: 'Datasets', component: DashboardDatasetsTab, diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index 7b70539a3b5a5..52c705e0a840d 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -27,6 +27,7 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; import { SidebarSiblingsSection } from '../shared/containers/profile/sidebar/SidebarSiblingsSection'; import { DatasetStatsSummarySubHeader } from './profile/stats/stats/DatasetStatsSummarySubHeader'; import { DatasetSearchSnippet } from './DatasetSearchSnippet'; +import { EmbedTab } from '../shared/tabs/Embed/EmbedTab'; const SUBTYPES = { VIEW: 'view', @@ -108,13 +109,21 @@ export class DatasetEntity implements Entity { component: DocumentationTab, }, { - name: 'Properties', - component: PropertiesTab, + name: 'Preview', + component: EmbedTab, + display: { + visible: (_, dataset: GetDatasetQuery) => !!dataset?.dataset?.embed?.renderUrl, + enabled: (_, dataset: GetDatasetQuery) => !!dataset?.dataset?.embed?.renderUrl, + }, }, { name: 'Lineage', component: LineageTab, }, + { + name: 'Properties', + component: PropertiesTab, + }, { name: 'Queries', component: QueriesTab, diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueriesTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueriesTab.tsx index 6424a831f74dc..7049e757b5512 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueriesTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueriesTab.tsx @@ -4,10 +4,16 @@ import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; import Query from './Query'; import { useBaseEntity } from '../../../EntityContext'; import getTopNQueries from './utils/getTopNQueries'; +import { useAppConfig } from '../../../../../useAppConfig'; export default function QueriesTab() { + const appConfig = useAppConfig(); const baseEntity = useBaseEntity(); - const topQueries = getTopNQueries(5, baseEntity?.dataset?.usageStats?.buckets); + + const topQueries = getTopNQueries( + appConfig?.config?.visualConfig?.queriesTab?.queriesTabResultSize || 5, + baseEntity?.dataset?.usageStats?.buckets, + ); if (topQueries.length === 0) { return ; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Embed/EmbedTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Embed/EmbedTab.tsx new file mode 100644 index 0000000000000..8732362b31afc --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Embed/EmbedTab.tsx @@ -0,0 +1,33 @@ +import { Empty } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import { ANTD_GRAY } from '../../constants'; +import { useEntityData } from '../../EntityContext'; + +const EmbedContainer = styled.div` + width: 100%; + height: 100%; +`; + +const StyledIframe = styled.iframe` + width: 100%; + height: 100%; +`; + +const StyledEmpty = styled(Empty)` + margin-top: 28px; + font-size: 16px; + color: ${ANTD_GRAY[8]}; +`; + +export const EmbedTab = () => { + const { entityData } = useEntityData(); + const embedRenderUrl = entityData?.embed?.renderUrl; + return ( + + {(embedRenderUrl && ) || ( + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index 7d448b667036d..41b7132dbbf8f 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -33,6 +33,7 @@ import { InputFields, FineGrainedLineage, EntityPrivileges, + Embed, } from '../../../types.generated'; import { FetchedEntity } from '../../lineage/types'; @@ -101,6 +102,7 @@ export type GenericEntityProperties = { inputFields?: Maybe; fineGrainedLineages?: Maybe; privileges?: Maybe; + embed?: Maybe; }; export type GenericEntityUpdate = { diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 7376fdbe2afe7..ac9d4adf3f5b6 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -21,6 +21,9 @@ export const DEFAULT_APP_CONFIG = { }, visualConfig: { logoUrl: undefined, + queriesTab: { + queriesTabResultSize: 5, + }, }, authConfig: { tokenAuthEnabled: false, diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index b09b27f0ed54d..fa5511530fd7d 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -37,6 +37,9 @@ query appConfig { visualConfig { logoUrl faviconUrl + queriesTab { + queriesTabResultSize + } } telemetryConfig { enableThirdPartyLogging diff --git a/datahub-web-react/src/graphql/chart.graphql b/datahub-web-react/src/graphql/chart.graphql index 1ae685bd67375..bb83beb69fd3d 100644 --- a/datahub-web-react/src/graphql/chart.graphql +++ b/datahub-web-react/src/graphql/chart.graphql @@ -51,6 +51,9 @@ query getChart($urn: String!) { deprecation { ...deprecationFields } + embed { + ...embedFields + } inputs: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) { ...fullRelationshipResults } @@ -96,6 +99,7 @@ query getChart($urn: String!) { } privileges { canEditLineage + canEditEmbed } } } diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 02601855a5b66..193dd7a8d7dec 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -207,10 +207,11 @@ fragment nonSiblingDatasetFields on Dataset { } } siblings { - isPrimary + isPrimary } privileges { canEditLineage + canEditEmbed } } diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index e1cea6568b2bb..a8b6f9574224d 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -172,6 +172,10 @@ fragment ownershipFields on Ownership { } } +fragment embedFields on Embed { + renderUrl +} + fragment institutionalMemoryFields on InstitutionalMemory { elements { url @@ -249,6 +253,9 @@ fragment nonRecursiveDatasetFields on Dataset { deprecation { ...deprecationFields } + embed { + ...embedFields + } } fragment nonRecursiveDataFlowFields on DataFlow { @@ -405,6 +412,9 @@ fragment dashboardFields on Dashboard { status { removed } + embed { + ...embedFields + } deprecation { ...deprecationFields } @@ -438,6 +448,7 @@ fragment dashboardFields on Dashboard { } privileges { canEditLineage + canEditEmbed } } @@ -768,7 +779,7 @@ fragment nonRecursiveMLModel on MLModel { } institutionalMemory { ...institutionalMemoryFields - } + } } fragment nonRecursiveMLModelGroupFields on MLModelGroup { diff --git a/datahub-web-react/src/graphql/mutations.graphql b/datahub-web-react/src/graphql/mutations.graphql index 71967afa7b68b..439d20810ef7c 100644 --- a/datahub-web-react/src/graphql/mutations.graphql +++ b/datahub-web-react/src/graphql/mutations.graphql @@ -123,3 +123,7 @@ mutation createPost($input: CreatePostInput!) { mutation updateLineage($input: UpdateLineageInput!) { updateLineage(input: $input) } + +mutation updateEmbed($input: UpdateEmbedInput!) { + updateEmbed(input: $input) +} diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index 3f6089dbe475d..85f19491123f6 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -2,6 +2,14 @@ # yarn lockfile v1 +"@ampproject/remapping@^2.1.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.0.tgz#56c133824780de3174aed5ab6834f3026790154d" + integrity sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w== + dependencies: + "@jridgewell/gen-mapping" "^0.1.0" + "@jridgewell/trace-mapping" "^0.3.9" + "@analytics/amplitude@0.0.3": version "0.0.3" resolved "https://registry.yarnpkg.com/@analytics/amplitude/-/amplitude-0.0.3.tgz#15ccb76094d6f1003979a4f3aa5d3263781bd776" @@ -26,23 +34,11 @@ dependencies: universal-analytics "^0.4.20" -"@analytics/google-tag-manager@^0.5.0": - version "0.5.0" - resolved "https://registry.yarnpkg.com/@analytics/google-tag-manager/-/google-tag-manager-0.5.0.tgz#172c90c0ef6a25f2dc1d06fd60d724dfd99afc3c" - integrity sha512-Md1qeNlEGViEEaljoYvKtGgRyi41v9VaKqoxCDXbcvssQCio/Ha0qcOiObtFZDPKfYFke8Fm+nzE6AjUcm2vuA== - "@analytics/mixpanel@^0.3.1": version "0.3.1" resolved "https://registry.yarnpkg.com/@analytics/mixpanel/-/mixpanel-0.3.1.tgz#d14264774302b1b2039115ebe3d5dbe838376081" integrity sha512-zYtzBUdcza/tAT1if6CN0s+BAqtrA2k17WoMFfi6nrLYEj8V60h/o3l+6RUDyIfIg1GkoUhiAJVxJvxW+CMnCw== -"@analytics/segment@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@analytics/segment/-/segment-1.1.0.tgz#5fe57bec0c71573ba024cdbb08c50fc31a4db105" - integrity sha512-y8cywcyCH9EjBXLGAPMK3LJJTG1ulPVig83i5FNphRFF+UJaAXcVe2LAeKZTGeyzUC00uYyf+LNkA58WAhj6pA== - dependencies: - analytics-node "^3.5.0" - "@analytics/storage-utils@^0.2.5": version "0.2.5" resolved "https://registry.yarnpkg.com/@analytics/storage-utils/-/storage-utils-0.2.5.tgz#bc82a1b5dea8e47ae18718aae687c3408ae7c504" @@ -133,6 +129,29 @@ dependencies: tslib "~2.0.1" +"@ardatan/relay-compiler@12.0.0": + version "12.0.0" + resolved "https://registry.yarnpkg.com/@ardatan/relay-compiler/-/relay-compiler-12.0.0.tgz#2e4cca43088e807adc63450e8cab037020e91106" + integrity sha512-9anThAaj1dQr6IGmzBMcfzOQKTa5artjuPmw8NYK/fiGEMjADbSguBY2FMDykt+QhilR3wc9VA/3yVju7JHg7Q== + dependencies: + "@babel/core" "^7.14.0" + "@babel/generator" "^7.14.0" + "@babel/parser" "^7.14.0" + "@babel/runtime" "^7.0.0" + "@babel/traverse" "^7.14.0" + "@babel/types" "^7.0.0" + babel-preset-fbjs "^3.4.0" + chalk "^4.0.0" + fb-watchman "^2.0.0" + fbjs "^3.0.0" + glob "^7.1.1" + immutable "~3.7.6" + invariant "^2.2.4" + nullthrows "^1.1.1" + relay-runtime "12.0.0" + signedsource "^1.0.0" + yargs "^15.3.1" + "@babel/code-frame@7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a" @@ -161,11 +180,23 @@ dependencies: "@babel/highlight" "^7.14.5" +"@babel/code-frame@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" + integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== + dependencies: + "@babel/highlight" "^7.18.6" + "@babel/compat-data@^7.12.1", "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.13.15", "@babel/compat-data@^7.14.0": version "7.14.0" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.0.tgz#a901128bce2ad02565df95e6ecbf195cf9465919" integrity sha512-vu9V3uMM/1o5Hl5OekMUowo3FqXLJSw+s+66nt0fSWVWTtmosdzn45JHOB3cPtZoe6CTBDzvSw0RdOY85Q37+Q== +"@babel/compat-data@^7.20.5": + version "7.20.10" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.20.10.tgz#9d92fa81b87542fff50e848ed585b4212c1d34ec" + integrity sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg== + "@babel/core@7.12.3": version "7.12.3" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.3.tgz#1b436884e1e3bff6fb1328dc02b208759de92ad8" @@ -188,7 +219,7 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.12.3", "@babel/core@^7.7.5", "@babel/core@^7.8.4": +"@babel/core@^7.1.0", "@babel/core@^7.12.3", "@babel/core@^7.7.5", "@babel/core@^7.8.4": version "7.14.3" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.14.3.tgz#5395e30405f0776067fbd9cf0884f15bfb770a38" integrity sha512-jB5AmTKOCSJIZ72sd78ECEhuPiDMKlQdDI/4QRI6lzYATx5SSogS1oQA2AoPecRCknm30gHi2l+QVvNUu3wZAg== @@ -209,7 +240,28 @@ semver "^6.3.0" source-map "^0.5.0" -"@babel/generator@^7.12.1", "@babel/generator@^7.12.13", "@babel/generator@^7.14.2", "@babel/generator@^7.14.3", "@babel/generator@^7.5.0": +"@babel/core@^7.14.0": + version "7.20.12" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.20.12.tgz#7930db57443c6714ad216953d1356dac0eb8496d" + integrity sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg== + dependencies: + "@ampproject/remapping" "^2.1.0" + "@babel/code-frame" "^7.18.6" + "@babel/generator" "^7.20.7" + "@babel/helper-compilation-targets" "^7.20.7" + "@babel/helper-module-transforms" "^7.20.11" + "@babel/helpers" "^7.20.7" + "@babel/parser" "^7.20.7" + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.20.12" + "@babel/types" "^7.20.7" + convert-source-map "^1.7.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.2.2" + semver "^6.3.0" + +"@babel/generator@^7.12.1", "@babel/generator@^7.12.13", "@babel/generator@^7.14.2", "@babel/generator@^7.14.3": version "7.14.3" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.14.3.tgz#0c2652d91f7bddab7cccc6ba8157e4f40dcedb91" integrity sha512-bn0S6flG/j0xtQdz3hsjJ624h3W0r3llttBMfyHX3YrZ/KtLYr15bjA0FXkgW7FpvrDuTuElXeVjiKlYRpnOFA== @@ -218,6 +270,15 @@ jsesc "^2.5.1" source-map "^0.5.0" +"@babel/generator@^7.14.0", "@babel/generator@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.7.tgz#f8ef57c8242665c5929fe2e8d82ba75460187b4a" + integrity sha512-7wqMOJq8doJMZmP4ApXTzLxSr7+oO2jroJURrVEp6XShrQUObV8Tq/D0NCcoYg2uHqUrjzO0zwBjoYzelxK+sw== + dependencies: + "@babel/types" "^7.20.7" + "@jridgewell/gen-mapping" "^0.3.2" + jsesc "^2.5.1" + "@babel/generator@^7.14.5": version "7.14.5" resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.14.5.tgz#848d7b9f031caca9d0cd0af01b063f226f52d785" @@ -234,6 +295,13 @@ dependencies: "@babel/types" "^7.12.13" +"@babel/helper-annotate-as-pure@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" + integrity sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA== + dependencies: + "@babel/types" "^7.18.6" + "@babel/helper-builder-binary-assignment-operator-visitor@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.12.13.tgz#6bc20361c88b0a74d05137a65cac8d3cbf6f61fc" @@ -252,6 +320,17 @@ browserslist "^4.14.5" semver "^6.3.0" +"@babel/helper-compilation-targets@^7.18.9", "@babel/helper-compilation-targets@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz#a6cd33e93629f5eb473b021aac05df62c4cd09bb" + integrity sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ== + dependencies: + "@babel/compat-data" "^7.20.5" + "@babel/helper-validator-option" "^7.18.6" + browserslist "^4.21.3" + lru-cache "^5.1.1" + semver "^6.3.0" + "@babel/helper-create-class-features-plugin@^7.12.1", "@babel/helper-create-class-features-plugin@^7.13.0", "@babel/helper-create-class-features-plugin@^7.14.0", "@babel/helper-create-class-features-plugin@^7.14.3": version "7.14.3" resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.3.tgz#832111bcf4f57ca57a4c5b1a000fc125abc6554a" @@ -264,6 +343,20 @@ "@babel/helper-replace-supers" "^7.14.3" "@babel/helper-split-export-declaration" "^7.12.13" +"@babel/helper-create-class-features-plugin@^7.18.6": + version "7.20.12" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.20.12.tgz#4349b928e79be05ed2d1643b20b99bb87c503819" + integrity sha512-9OunRkbT0JQcednL0UFvbfXpAsUXiGjUk0a7sN8fUXX7Mue79cUSMjHGDRRi/Vz9vYlpIhLV5fMD5dKoMhhsNQ== + dependencies: + "@babel/helper-annotate-as-pure" "^7.18.6" + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-function-name" "^7.19.0" + "@babel/helper-member-expression-to-functions" "^7.20.7" + "@babel/helper-optimise-call-expression" "^7.18.6" + "@babel/helper-replace-supers" "^7.20.7" + "@babel/helper-skip-transparent-expression-wrappers" "^7.20.0" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/helper-create-regexp-features-plugin@^7.12.13": version "7.14.3" resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.14.3.tgz#149aa6d78c016e318c43e2409a0ae9c136a86688" @@ -286,6 +379,11 @@ resolve "^1.14.2" semver "^6.1.2" +"@babel/helper-environment-visitor@^7.18.9": + version "7.18.9" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz#0c0cee9b35d2ca190478756865bb3528422f51be" + integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg== + "@babel/helper-explode-assignable-expression@^7.12.13": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.13.0.tgz#17b5c59ff473d9f956f40ef570cf3a76ca12657f" @@ -311,6 +409,14 @@ "@babel/template" "^7.14.5" "@babel/types" "^7.14.5" +"@babel/helper-function-name@^7.18.9", "@babel/helper-function-name@^7.19.0": + version "7.19.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz#941574ed5390682e872e52d3f38ce9d1bef4648c" + integrity sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w== + dependencies: + "@babel/template" "^7.18.10" + "@babel/types" "^7.19.0" + "@babel/helper-get-function-arity@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583" @@ -340,6 +446,13 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-hoist-variables@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678" + integrity sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q== + dependencies: + "@babel/types" "^7.18.6" + "@babel/helper-member-expression-to-functions@^7.13.12": version "7.13.12" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.13.12.tgz#dfe368f26d426a07299d8d6513821768216e6d72" @@ -347,6 +460,13 @@ dependencies: "@babel/types" "^7.13.12" +"@babel/helper-member-expression-to-functions@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.20.7.tgz#a6f26e919582275a93c3aa6594756d71b0bb7f05" + integrity sha512-9J0CxJLq315fEdi4s7xK5TQaNYjZw+nDVpVqr1axNGKzdrdwYBD5b4uKv3n75aABG0rCCTK8Im8Ww7eYfMrZgw== + dependencies: + "@babel/types" "^7.20.7" + "@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.0.0-beta.49", "@babel/helper-module-imports@^7.12.1", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.13.12": version "7.13.12" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.13.12.tgz#c6a369a6f3621cb25da014078684da9196b61977" @@ -354,6 +474,13 @@ dependencies: "@babel/types" "^7.13.12" +"@babel/helper-module-imports@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz#1e3ebdbbd08aad1437b428c50204db13c5a3ca6e" + integrity sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA== + dependencies: + "@babel/types" "^7.18.6" + "@babel/helper-module-transforms@^7.12.1", "@babel/helper-module-transforms@^7.13.0", "@babel/helper-module-transforms@^7.14.0", "@babel/helper-module-transforms@^7.14.2": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.14.2.tgz#ac1cc30ee47b945e3e0c4db12fa0c5389509dfe5" @@ -368,6 +495,20 @@ "@babel/traverse" "^7.14.2" "@babel/types" "^7.14.2" +"@babel/helper-module-transforms@^7.20.11": + version "7.20.11" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.20.11.tgz#df4c7af713c557938c50ea3ad0117a7944b2f1b0" + integrity sha512-uRy78kN4psmji1s2QtbtcCSaj/LILFDp0f/ymhpQH5QY3nljUZCaNWz9X1dEj/8MBdBEFECs7yRhKn8i7NjZgg== + dependencies: + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-module-imports" "^7.18.6" + "@babel/helper-simple-access" "^7.20.2" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/helper-validator-identifier" "^7.19.1" + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.20.10" + "@babel/types" "^7.20.7" + "@babel/helper-optimise-call-expression@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.13.tgz#5c02d171b4c8615b1e7163f888c1c81c30a2aaea" @@ -375,11 +516,23 @@ dependencies: "@babel/types" "^7.12.13" +"@babel/helper-optimise-call-expression@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz#9369aa943ee7da47edab2cb4e838acf09d290ffe" + integrity sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA== + dependencies: + "@babel/types" "^7.18.6" + "@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.13.0", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.13.0.tgz#806526ce125aed03373bc416a828321e3a6a33af" integrity sha512-ZPafIPSwzUlAoWT8DKs1W2VyF2gOWthGd5NGFMsBcMMol+ZhK+EQY/e6V96poa6PA/Bh+C9plWN0hXO1uB8AfQ== +"@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.18.9", "@babel/helper-plugin-utils@^7.19.0", "@babel/helper-plugin-utils@^7.20.2": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz#d1b9000752b18d0877cff85a5c376ce5c3121629" + integrity sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ== + "@babel/helper-remap-async-to-generator@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.13.0.tgz#376a760d9f7b4b2077a9dd05aa9c3927cadb2209" @@ -399,6 +552,18 @@ "@babel/traverse" "^7.14.2" "@babel/types" "^7.14.2" +"@babel/helper-replace-supers@^7.18.6", "@babel/helper-replace-supers@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.20.7.tgz#243ecd2724d2071532b2c8ad2f0f9f083bcae331" + integrity sha512-vujDMtB6LVfNW13jhlCrp48QNslK6JXi7lQG736HVbHz/mbf4Dc7tIRh1Xf5C0rF7BP8iiSxGMCmY6Ci1ven3A== + dependencies: + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-member-expression-to-functions" "^7.20.7" + "@babel/helper-optimise-call-expression" "^7.18.6" + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.20.7" + "@babel/types" "^7.20.7" + "@babel/helper-simple-access@^7.13.12": version "7.13.12" resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.13.12.tgz#dd6c538afb61819d205a012c31792a39c7a5eaf6" @@ -406,6 +571,13 @@ dependencies: "@babel/types" "^7.13.12" +"@babel/helper-simple-access@^7.20.2": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz#0ab452687fe0c2cfb1e2b9e0015de07fc2d62dd9" + integrity sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA== + dependencies: + "@babel/types" "^7.20.2" + "@babel/helper-skip-transparent-expression-wrappers@^7.12.1": version "7.12.1" resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.12.1.tgz#462dc63a7e435ade8468385c63d2b84cce4b3cbf" @@ -413,6 +585,13 @@ dependencies: "@babel/types" "^7.12.1" +"@babel/helper-skip-transparent-expression-wrappers@^7.20.0": + version "7.20.0" + resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.20.0.tgz#fbe4c52f60518cab8140d77101f0e63a8a230684" + integrity sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg== + dependencies: + "@babel/types" "^7.20.0" + "@babel/helper-split-export-declaration@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05" @@ -427,6 +606,18 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-split-export-declaration@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz#7367949bc75b20c6d5a5d4a97bba2824ae8ef075" + integrity sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA== + dependencies: + "@babel/types" "^7.18.6" + +"@babel/helper-string-parser@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63" + integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== + "@babel/helper-validator-identifier@^7.12.11", "@babel/helper-validator-identifier@^7.14.0": version "7.14.0" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.0.tgz#d26cad8a47c65286b15df1547319a5d0bcf27288" @@ -437,11 +628,21 @@ resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8" integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg== +"@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": + version "7.19.1" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" + integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== + "@babel/helper-validator-option@^7.12.1", "@babel/helper-validator-option@^7.12.17": version "7.12.17" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.12.17.tgz#d1fbf012e1a79b7eebbfdc6d270baaf8d9eb9831" integrity sha512-TopkMDmLzq8ngChwRlyjR6raKD6gMSae4JdYDB8bByKreQgG0RBTuKe9LRxW3wFtUnjxOPRKBDwEH6Mg5KeDfw== +"@babel/helper-validator-option@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz#bf0d2b5a509b1f336099e4ff36e1a63aa5db4db8" + integrity sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw== + "@babel/helper-wrap-function@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.13.0.tgz#bdb5c66fda8526ec235ab894ad53a1235c79fcc4" @@ -461,6 +662,15 @@ "@babel/traverse" "^7.14.0" "@babel/types" "^7.14.0" +"@babel/helpers@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.20.7.tgz#04502ff0feecc9f20ecfaad120a18f011a8e6dce" + integrity sha512-PBPjs5BppzsGaxHQCDKnZ6Gd9s6xl8bBCluz3vEInLGRJmnZan4F6BYCeqtyXqkk4W5IlPmjK4JlOuZkpJ3xZA== + dependencies: + "@babel/template" "^7.20.7" + "@babel/traverse" "^7.20.7" + "@babel/types" "^7.20.7" + "@babel/highlight@^7.10.4", "@babel/highlight@^7.12.13": version "7.14.0" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.14.0.tgz#3197e375711ef6bf834e67d0daec88e4f46113cf" @@ -479,12 +689,21 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.18.6.tgz#81158601e93e2563795adcbfbdf5d64be3f2ecdf" + integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g== + dependencies: + "@babel/helper-validator-identifier" "^7.18.6" + chalk "^2.0.0" + js-tokens "^4.0.0" + "@babel/parser@7.12.16": version "7.12.16" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.16.tgz#cc31257419d2c3189d394081635703f549fc1ed4" integrity sha512-c/+u9cqV6F0+4Hpq01jnJO+GLp2DdT63ppz9Xa+6cHaajM9VFzK/iDXiKK65YtpeVwu+ctfS6iqlMqRgQRzeCw== -"@babel/parser@^7.0.0", "@babel/parser@^7.1.0", "@babel/parser@^7.12.13", "@babel/parser@^7.12.3", "@babel/parser@^7.14.2", "@babel/parser@^7.14.3", "@babel/parser@^7.7.0": +"@babel/parser@^7.1.0", "@babel/parser@^7.12.13", "@babel/parser@^7.12.3", "@babel/parser@^7.14.2", "@babel/parser@^7.14.3", "@babel/parser@^7.7.0": version "7.14.3" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.3.tgz#9b530eecb071fd0c93519df25c5ff9f14759f298" integrity sha512-7MpZDIfI7sUC5zWo2+foJ50CSI5lcqDehZ0lVgIhSi4bFEk94fLAKlF3Q0nzSQQ+ca0lm+O6G9ztKVBeu8PMRQ== @@ -494,6 +713,11 @@ resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.14.7.tgz#6099720c8839ca865a2637e6c85852ead0bdb595" integrity sha512-X67Z5y+VBJuHB/RjwECp8kSl5uYi0BvRbNeWqkaJCVh+LiTPl19WBUfG627psSgp9rSf6ojuXghQM3ha6qHHdA== +"@babel/parser@^7.14.0", "@babel/parser@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.7.tgz#66fe23b3c8569220817d5feb8b9dcdc95bb4f71b" + integrity sha512-T3Z9oHybU+0vZlY9CiDSJQTD5ZapcW18ZctFMi0MOAl/4BjFF4ul7NVSARLdbGO5vDqy9eQiGTV0LtKfvCYvcg== + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.13.12": version "7.13.12" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.13.12.tgz#a3484d84d0b549f3fc916b99ee4783f26fabad2a" @@ -520,7 +744,15 @@ "@babel/helper-create-class-features-plugin" "^7.12.1" "@babel/helper-plugin-utils" "^7.10.4" -"@babel/plugin-proposal-class-properties@^7.0.0", "@babel/plugin-proposal-class-properties@^7.12.1", "@babel/plugin-proposal-class-properties@^7.13.0": +"@babel/plugin-proposal-class-properties@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz#b110f59741895f7ec21a6fff696ec46265c446a3" + integrity sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ== + dependencies: + "@babel/helper-create-class-features-plugin" "^7.18.6" + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-proposal-class-properties@^7.12.1", "@babel/plugin-proposal-class-properties@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.13.0.tgz#146376000b94efd001e57a40a88a525afaab9f37" integrity sha512-KnTDjFNC1g+45ka0myZNvSBFLhNCLN+GeGYLDEA8Oq7MZ6yMgfLoIRh86GRT0FjtJhZw8JyUskP9uvj5pHM9Zg== @@ -610,7 +842,18 @@ "@babel/helper-plugin-utils" "^7.13.0" "@babel/plugin-syntax-numeric-separator" "^7.10.4" -"@babel/plugin-proposal-object-rest-spread@^7.0.0", "@babel/plugin-proposal-object-rest-spread@^7.12.1", "@babel/plugin-proposal-object-rest-spread@^7.14.2": +"@babel/plugin-proposal-object-rest-spread@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.7.tgz#aa662940ef425779c75534a5c41e9d936edc390a" + integrity sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg== + dependencies: + "@babel/compat-data" "^7.20.5" + "@babel/helper-compilation-targets" "^7.20.7" + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/plugin-syntax-object-rest-spread" "^7.8.3" + "@babel/plugin-transform-parameters" "^7.20.7" + +"@babel/plugin-proposal-object-rest-spread@^7.12.1", "@babel/plugin-proposal-object-rest-spread@^7.14.2": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.14.2.tgz#e17d418f81cc103fedd4ce037e181c8056225abc" integrity sha512-hBIQFxwZi8GIp934+nj5uV31mqclC1aYDhctDu5khTi9PCCUOczyy0b34W0oE9U/eJXiqQaKyVsmjeagOaSlbw== @@ -722,7 +965,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.3" -"@babel/plugin-syntax-flow@^7.0.0", "@babel/plugin-syntax-flow@^7.12.1", "@babel/plugin-syntax-flow@^7.12.13": +"@babel/plugin-syntax-flow@^7.0.0", "@babel/plugin-syntax-flow@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz#774d825256f2379d06139be0c723c4dd444f3ca1" + integrity sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-syntax-flow@^7.12.1": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.12.13.tgz#5df9962503c0a9c918381c929d51d4d6949e7e86" integrity sha512-J/RYxnlSLXZLVR7wTRsozxKT8qbsx1mNKJzXEEjQ0Kjx1ZACcyHgbanNWNCFtc36IzuWhYWPpvJFFoexoOWFmA== @@ -743,7 +993,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-jsx@^7.0.0", "@babel/plugin-syntax-jsx@^7.12.13": +"@babel/plugin-syntax-jsx@^7.0.0", "@babel/plugin-syntax-jsx@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.18.6.tgz#a8feef63b010150abd97f1649ec296e849943ca0" + integrity sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-syntax-jsx@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.13.tgz#044fb81ebad6698fe62c478875575bcbb9b70f15" integrity sha512-d4HM23Q1K7oq/SLNmG6mRt85l2csmQ0cHRaxRXjKW0YFdEXqlZ5kzFQKH5Uc3rDJECgu+yCRgPkG04Mm98R/1g== @@ -813,7 +1070,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-arrow-functions@^7.0.0", "@babel/plugin-transform-arrow-functions@^7.12.1", "@babel/plugin-transform-arrow-functions@^7.13.0": +"@babel/plugin-transform-arrow-functions@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.20.7.tgz#bea332b0e8b2dab3dafe55a163d8227531ab0551" + integrity sha512-3poA5E7dzDomxj9WXWwuD6A5F3kc7VXwIJO+E+J8qtDtS+pXPAhrgEyh+9GBwBgPq1Z+bB+/JD60lp5jsN7JPQ== + dependencies: + "@babel/helper-plugin-utils" "^7.20.2" + +"@babel/plugin-transform-arrow-functions@^7.12.1", "@babel/plugin-transform-arrow-functions@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.13.0.tgz#10a59bebad52d637a027afa692e8d5ceff5e3dae" integrity sha512-96lgJagobeVmazXFaDrbmCLQxBysKu7U6Do3mLsx27gf5Dk85ezysrs2BZUpXD703U/Su1xTBDxxar2oa4jAGg== @@ -829,21 +1093,50 @@ "@babel/helper-plugin-utils" "^7.13.0" "@babel/helper-remap-async-to-generator" "^7.13.0" -"@babel/plugin-transform-block-scoped-functions@^7.0.0", "@babel/plugin-transform-block-scoped-functions@^7.12.1", "@babel/plugin-transform-block-scoped-functions@^7.12.13": +"@babel/plugin-transform-block-scoped-functions@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz#9187bf4ba302635b9d70d986ad70f038726216a8" + integrity sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-transform-block-scoped-functions@^7.12.1", "@babel/plugin-transform-block-scoped-functions@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.12.13.tgz#a9bf1836f2a39b4eb6cf09967739de29ea4bf4c4" integrity sha512-zNyFqbc3kI/fVpqwfqkg6RvBgFpC4J18aKKMmv7KdQ/1GgREapSJAykLMVNwfRGO3BtHj3YQZl8kxCXPcVMVeg== dependencies: "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-block-scoping@^7.0.0", "@babel/plugin-transform-block-scoping@^7.12.1", "@babel/plugin-transform-block-scoping@^7.14.2": +"@babel/plugin-transform-block-scoping@^7.0.0": + version "7.20.11" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.20.11.tgz#9f5a3424bd112a3f32fe0cf9364fbb155cff262a" + integrity sha512-tA4N427a7fjf1P0/2I4ScsHGc5jcHPbb30xMbaTke2gxDuWpUfXDuX1FEymJwKk4tuGUvGcejAR6HdZVqmmPyw== + dependencies: + "@babel/helper-plugin-utils" "^7.20.2" + +"@babel/plugin-transform-block-scoping@^7.12.1", "@babel/plugin-transform-block-scoping@^7.14.2": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.14.2.tgz#761cb12ab5a88d640ad4af4aa81f820e6b5fdf5c" integrity sha512-neZZcP19NugZZqNwMTH+KoBjx5WyvESPSIOQb4JHpfd+zPfqcH65RMu5xJju5+6q/Y2VzYrleQTr+b6METyyxg== dependencies: "@babel/helper-plugin-utils" "^7.13.0" -"@babel/plugin-transform-classes@^7.0.0", "@babel/plugin-transform-classes@^7.12.1", "@babel/plugin-transform-classes@^7.14.2": +"@babel/plugin-transform-classes@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.20.7.tgz#f438216f094f6bb31dc266ebfab8ff05aecad073" + integrity sha512-LWYbsiXTPKl+oBlXUGlwNlJZetXD5Am+CyBdqhPsDVjM9Jc8jwBJFrKhHf900Kfk2eZG1y9MAG3UNajol7A4VQ== + dependencies: + "@babel/helper-annotate-as-pure" "^7.18.6" + "@babel/helper-compilation-targets" "^7.20.7" + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-function-name" "^7.19.0" + "@babel/helper-optimise-call-expression" "^7.18.6" + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/helper-replace-supers" "^7.20.7" + "@babel/helper-split-export-declaration" "^7.18.6" + globals "^11.1.0" + +"@babel/plugin-transform-classes@^7.12.1", "@babel/plugin-transform-classes@^7.14.2": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.14.2.tgz#3f1196c5709f064c252ad056207d87b7aeb2d03d" integrity sha512-7oafAVcucHquA/VZCsXv/gmuiHeYd64UJyyTYU+MPfNu0KeNlxw06IeENBO8bJjXVbolu+j1MM5aKQtH1OMCNg== @@ -856,14 +1149,29 @@ "@babel/helper-split-export-declaration" "^7.12.13" globals "^11.1.0" -"@babel/plugin-transform-computed-properties@^7.0.0", "@babel/plugin-transform-computed-properties@^7.12.1", "@babel/plugin-transform-computed-properties@^7.13.0": +"@babel/plugin-transform-computed-properties@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.20.7.tgz#704cc2fd155d1c996551db8276d55b9d46e4d0aa" + integrity sha512-Lz7MvBK6DTjElHAmfu6bfANzKcxpyNPeYBGEafyA6E5HtRpjpZwU+u7Qrgz/2OR0z+5TvKYbPdphfSaAcZBrYQ== + dependencies: + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/template" "^7.20.7" + +"@babel/plugin-transform-computed-properties@^7.12.1", "@babel/plugin-transform-computed-properties@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.13.0.tgz#845c6e8b9bb55376b1fa0b92ef0bdc8ea06644ed" integrity sha512-RRqTYTeZkZAz8WbieLTvKUEUxZlUTdmL5KGMyZj7FnMfLNKV4+r5549aORG/mgojRmFlQMJDUupwAMiF2Q7OUg== dependencies: "@babel/helper-plugin-utils" "^7.13.0" -"@babel/plugin-transform-destructuring@^7.0.0", "@babel/plugin-transform-destructuring@^7.12.1", "@babel/plugin-transform-destructuring@^7.13.17": +"@babel/plugin-transform-destructuring@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.20.7.tgz#8bda578f71620c7de7c93af590154ba331415454" + integrity sha512-Xwg403sRrZb81IVB79ZPqNQME23yhugYVqgTxAhT99h485F4f+GMELFhhOsscDUB7HCswepKeCKLn/GZvUKoBA== + dependencies: + "@babel/helper-plugin-utils" "^7.20.2" + +"@babel/plugin-transform-destructuring@^7.12.1", "@babel/plugin-transform-destructuring@^7.13.17": version "7.13.17" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.13.17.tgz#678d96576638c19d5b36b332504d3fd6e06dea27" integrity sha512-UAUqiLv+uRLO+xuBKKMEpC+t7YRNVRqBsWWq1yKXbBZBje/t3IXCiSinZhjn/DC3qzBfICeYd2EFGEbHsh5RLA== @@ -902,21 +1210,37 @@ "@babel/plugin-syntax-flow" "^7.12.1" "@babel/plugin-transform-flow-strip-types@^7.0.0": - version "7.13.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.13.0.tgz#58177a48c209971e8234e99906cb6bd1122addd3" - integrity sha512-EXAGFMJgSX8gxWD7PZtW/P6M+z74jpx3wm/+9pn+c2dOawPpBkUX7BrfyPvo6ZpXbgRIEuwgwDb/MGlKvu2pOg== + version "7.19.0" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.19.0.tgz#e9e8606633287488216028719638cbbb2f2dde8f" + integrity sha512-sgeMlNaQVbCSpgLSKP4ZZKfsJVnFnNQlUSk6gPYzR/q7tzCgQF2t8RBKAP6cKJeZdveei7Q7Jm527xepI8lNLg== dependencies: - "@babel/helper-plugin-utils" "^7.13.0" - "@babel/plugin-syntax-flow" "^7.12.13" + "@babel/helper-plugin-utils" "^7.19.0" + "@babel/plugin-syntax-flow" "^7.18.6" + +"@babel/plugin-transform-for-of@^7.0.0": + version "7.18.8" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.8.tgz#6ef8a50b244eb6a0bdbad0c7c61877e4e30097c1" + integrity sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" -"@babel/plugin-transform-for-of@^7.0.0", "@babel/plugin-transform-for-of@^7.12.1", "@babel/plugin-transform-for-of@^7.13.0": +"@babel/plugin-transform-for-of@^7.12.1", "@babel/plugin-transform-for-of@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.13.0.tgz#c799f881a8091ac26b54867a845c3e97d2696062" integrity sha512-IHKT00mwUVYE0zzbkDgNRP6SRzvfGCYsOxIRz8KsiaaHCcT9BWIkO+H9QRJseHBLOGBZkHUdHiqj6r0POsdytg== dependencies: "@babel/helper-plugin-utils" "^7.13.0" -"@babel/plugin-transform-function-name@^7.0.0", "@babel/plugin-transform-function-name@^7.12.1", "@babel/plugin-transform-function-name@^7.12.13": +"@babel/plugin-transform-function-name@^7.0.0": + version "7.18.9" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.9.tgz#cc354f8234e62968946c61a46d6365440fc764e0" + integrity sha512-WvIBoRPaJQ5yVHzcnJFor7oS5Ls0PYixlTYE63lCj2RtdQEl15M68FXQlxnG6wdraJIXRdR7KI+hQ7q/9QjrCQ== + dependencies: + "@babel/helper-compilation-targets" "^7.18.9" + "@babel/helper-function-name" "^7.18.9" + "@babel/helper-plugin-utils" "^7.18.9" + +"@babel/plugin-transform-function-name@^7.12.1", "@babel/plugin-transform-function-name@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.12.13.tgz#bb024452f9aaed861d374c8e7a24252ce3a50051" integrity sha512-6K7gZycG0cmIwwF7uMK/ZqeCikCGVBdyP2J5SKNCXO5EOHcqi+z7Jwf8AmyDNcBgxET8DrEtCt/mPKPyAzXyqQ== @@ -924,14 +1248,28 @@ "@babel/helper-function-name" "^7.12.13" "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-literals@^7.0.0", "@babel/plugin-transform-literals@^7.12.1", "@babel/plugin-transform-literals@^7.12.13": +"@babel/plugin-transform-literals@^7.0.0": + version "7.18.9" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.9.tgz#72796fdbef80e56fba3c6a699d54f0de557444bc" + integrity sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg== + dependencies: + "@babel/helper-plugin-utils" "^7.18.9" + +"@babel/plugin-transform-literals@^7.12.1", "@babel/plugin-transform-literals@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.12.13.tgz#2ca45bafe4a820197cf315794a4d26560fe4bdb9" integrity sha512-FW+WPjSR7hiUxMcKqyNjP05tQ2kmBCdpEpZHY1ARm96tGQCCBvXKnpjILtDplUnJ/eHZ0lALLM+d2lMFSpYJrQ== dependencies: "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-member-expression-literals@^7.0.0", "@babel/plugin-transform-member-expression-literals@^7.12.1", "@babel/plugin-transform-member-expression-literals@^7.12.13": +"@babel/plugin-transform-member-expression-literals@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz#ac9fdc1a118620ac49b7e7a5d2dc177a1bfee88e" + integrity sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-transform-member-expression-literals@^7.12.1", "@babel/plugin-transform-member-expression-literals@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.12.13.tgz#5ffa66cd59b9e191314c9f1f803b938e8c081e40" integrity sha512-kxLkOsg8yir4YeEPHLuO2tXP9R/gTjpuTOjshqSpELUN3ZAg2jfDnKUvzzJxObun38sw3wm4Uu69sX/zA7iRvg== @@ -947,7 +1285,16 @@ "@babel/helper-plugin-utils" "^7.13.0" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-commonjs@^7.0.0", "@babel/plugin-transform-modules-commonjs@^7.12.1", "@babel/plugin-transform-modules-commonjs@^7.14.0": +"@babel/plugin-transform-modules-commonjs@^7.0.0": + version "7.20.11" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.20.11.tgz#8cb23010869bf7669fd4b3098598b6b2be6dc607" + integrity sha512-S8e1f7WQ7cimJQ51JkAaDrEtohVEitXjgCGAS2N8S31Y42E+kWwfSz83LYz57QdBm7q9diARVqanIaH2oVgQnw== + dependencies: + "@babel/helper-module-transforms" "^7.20.11" + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/helper-simple-access" "^7.20.2" + +"@babel/plugin-transform-modules-commonjs@^7.12.1", "@babel/plugin-transform-modules-commonjs@^7.14.0": version "7.14.0" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.14.0.tgz#52bc199cb581e0992edba0f0f80356467587f161" integrity sha512-EX4QePlsTaRZQmw9BsoPeyh5OCtRGIhwfLquhxGp5e32w+dyL8htOcDwamlitmNFK6xBZYlygjdye9dbd9rUlQ== @@ -990,7 +1337,15 @@ dependencies: "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-object-super@^7.0.0", "@babel/plugin-transform-object-super@^7.12.1", "@babel/plugin-transform-object-super@^7.12.13": +"@babel/plugin-transform-object-super@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz#fb3c6ccdd15939b6ff7939944b51971ddc35912c" + integrity sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + "@babel/helper-replace-supers" "^7.18.6" + +"@babel/plugin-transform-object-super@^7.12.1", "@babel/plugin-transform-object-super@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.12.13.tgz#b4416a2d63b8f7be314f3d349bd55a9c1b5171f7" integrity sha512-JzYIcj3XtYspZDV8j9ulnoMPZZnF/Cj0LUxPOjR89BdBVx+zYJI9MdMIlUZjbXDX+6YVeS6I3e8op+qQ3BYBoQ== @@ -998,14 +1353,28 @@ "@babel/helper-plugin-utils" "^7.12.13" "@babel/helper-replace-supers" "^7.12.13" -"@babel/plugin-transform-parameters@^7.0.0", "@babel/plugin-transform-parameters@^7.12.1", "@babel/plugin-transform-parameters@^7.14.2": +"@babel/plugin-transform-parameters@^7.0.0", "@babel/plugin-transform-parameters@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.20.7.tgz#0ee349e9d1bc96e78e3b37a7af423a4078a7083f" + integrity sha512-WiWBIkeHKVOSYPO0pWkxGPfKeWrCJyD3NJ53+Lrp/QMSZbsVPovrVl2aWZ19D/LTVnaDv5Ap7GJ/B2CTOZdrfA== + dependencies: + "@babel/helper-plugin-utils" "^7.20.2" + +"@babel/plugin-transform-parameters@^7.12.1", "@babel/plugin-transform-parameters@^7.14.2": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.14.2.tgz#e4290f72e0e9e831000d066427c4667098decc31" integrity sha512-NxoVmA3APNCC1JdMXkdYXuQS+EMdqy0vIwyDHeKHiJKRxmp1qGSdb0JLEIoPRhkx6H/8Qi3RJ3uqOCYw8giy9A== dependencies: "@babel/helper-plugin-utils" "^7.13.0" -"@babel/plugin-transform-property-literals@^7.0.0", "@babel/plugin-transform-property-literals@^7.12.1", "@babel/plugin-transform-property-literals@^7.12.13": +"@babel/plugin-transform-property-literals@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz#e22498903a483448e94e032e9bbb9c5ccbfc93a3" + integrity sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-transform-property-literals@^7.12.1", "@babel/plugin-transform-property-literals@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.12.13.tgz#4e6a9e37864d8f1b3bc0e2dce7bf8857db8b1a81" integrity sha512-nqVigwVan+lR+g8Fj8Exl0UQX2kymtjcWfMOYM1vTYEKujeyv2SkMgazf2qNcK7l4SDiKyTA/nHCPqL4e2zo1A== @@ -1026,7 +1395,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.4" -"@babel/plugin-transform-react-display-name@^7.0.0", "@babel/plugin-transform-react-display-name@^7.12.1", "@babel/plugin-transform-react-display-name@^7.12.13": +"@babel/plugin-transform-react-display-name@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.18.6.tgz#8b1125f919ef36ebdfff061d664e266c666b9415" + integrity sha512-TV4sQ+T013n61uMoygyMRm+xf04Bd5oqFpv2jAEQwSZ8NwQA7zeRPg1LMVg2PWi3zWBz+CLKD+v5bcpZ/BS0aA== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-transform-react-display-name@^7.12.1", "@babel/plugin-transform-react-display-name@^7.12.13": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.14.2.tgz#2e854544d42ab3bb9c21f84e153d62e800fbd593" integrity sha512-zCubvP+jjahpnFJvPaHPiGVfuVUjXHhFvJKQdNnsmSsiU9kR/rCZ41jHc++tERD2zV+p7Hr6is+t5b6iWTCqSw== @@ -1054,7 +1430,18 @@ dependencies: "@babel/helper-plugin-utils" "^7.13.0" -"@babel/plugin-transform-react-jsx@^7.0.0", "@babel/plugin-transform-react-jsx@^7.12.1", "@babel/plugin-transform-react-jsx@^7.12.17", "@babel/plugin-transform-react-jsx@^7.13.12": +"@babel/plugin-transform-react-jsx@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.7.tgz#025d85a1935fd7e19dfdcb1b1d4df34d4da484f7" + integrity sha512-Tfq7qqD+tRj3EoDhY00nn2uP2hsRxgYGi5mLQ5TimKav0a9Lrpd4deE+fcLXU8zFYRjlKPHZhpCvfEA6qnBxqQ== + dependencies: + "@babel/helper-annotate-as-pure" "^7.18.6" + "@babel/helper-module-imports" "^7.18.6" + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/plugin-syntax-jsx" "^7.18.6" + "@babel/types" "^7.20.7" + +"@babel/plugin-transform-react-jsx@^7.12.1", "@babel/plugin-transform-react-jsx@^7.12.17", "@babel/plugin-transform-react-jsx@^7.13.12": version "7.14.3" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.14.3.tgz#0e26597805cf0862da735f264550933c38babb66" integrity sha512-uuxuoUNVhdgYzERiHHFkE4dWoJx+UFVyuAl0aqN8P2/AKFHwqgUC5w2+4/PjpKXJsFgBlYAFXlUmDQ3k3DUkXw== @@ -1097,14 +1484,29 @@ resolve "^1.8.1" semver "^5.5.1" -"@babel/plugin-transform-shorthand-properties@^7.0.0", "@babel/plugin-transform-shorthand-properties@^7.12.1", "@babel/plugin-transform-shorthand-properties@^7.12.13": +"@babel/plugin-transform-shorthand-properties@^7.0.0": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz#6d6df7983d67b195289be24909e3f12a8f664dc9" + integrity sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw== + dependencies: + "@babel/helper-plugin-utils" "^7.18.6" + +"@babel/plugin-transform-shorthand-properties@^7.12.1", "@babel/plugin-transform-shorthand-properties@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.12.13.tgz#db755732b70c539d504c6390d9ce90fe64aff7ad" integrity sha512-xpL49pqPnLtf0tVluuqvzWIgLEhuPpZzvs2yabUHSKRNlN7ScYU7aMlmavOeyXJZKgZKQRBlh8rHbKiJDraTSw== dependencies: "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-spread@^7.0.0", "@babel/plugin-transform-spread@^7.12.1", "@babel/plugin-transform-spread@^7.13.0": +"@babel/plugin-transform-spread@^7.0.0": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.20.7.tgz#c2d83e0b99d3bf83e07b11995ee24bf7ca09401e" + integrity sha512-ewBbHQ+1U/VnH1fxltbJqDeWBU1oNLG8Dj11uIv3xVf7nrQu0bPGe5Rf716r7K5Qz+SqtAOVswoVunoiBtGhxw== + dependencies: + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/helper-skip-transparent-expression-wrappers" "^7.20.0" + +"@babel/plugin-transform-spread@^7.12.1", "@babel/plugin-transform-spread@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.13.0.tgz#84887710e273c1815ace7ae459f6f42a5d31d5fd" integrity sha512-V6vkiXijjzYeFmQTr3dBxPtZYLPcUfY34DebOU27jIl2M/Y8Egm52Hw82CSjjPqd54GTlJs5x+CR7HeNr24ckg== @@ -1119,7 +1521,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-transform-template-literals@^7.0.0", "@babel/plugin-transform-template-literals@^7.12.1", "@babel/plugin-transform-template-literals@^7.13.0": +"@babel/plugin-transform-template-literals@^7.0.0": + version "7.18.9" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.9.tgz#04ec6f10acdaa81846689d63fae117dd9c243a5e" + integrity sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA== + dependencies: + "@babel/helper-plugin-utils" "^7.18.9" + +"@babel/plugin-transform-template-literals@^7.12.1", "@babel/plugin-transform-template-literals@^7.13.0": version "7.13.0" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.13.0.tgz#a36049127977ad94438dee7443598d1cefdf409d" integrity sha512-d67umW6nlfmr1iehCcBv69eSUSySk1EsIS8aTDX4Xo9qajAh6mYtcl4kJrBkGXuxZPEgVr7RVfAvNW6YQkd4Mw== @@ -1381,13 +1790,6 @@ dependencies: regenerator-runtime "^0.13.4" -"@babel/runtime@^7.12.0", "@babel/runtime@^7.13.10", "@babel/runtime@^7.8.7": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.16.3.tgz#b86f0db02a04187a3c17caa77de69840165d42d5" - integrity sha512-WBwekcqacdY2e9AF/Q7WLFUWmdJGJTkbjqTjoMDgXkVZ3ZRUvOPsLb5KdwISoQVsbP+DQzVZW4Zhci0DvpbNTQ== - dependencies: - regenerator-runtime "^0.13.4" - "@babel/runtime@^7.12.13": version "7.20.1" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.1.tgz#1148bb33ab252b165a06698fde7576092a78b4a9" @@ -1427,6 +1829,15 @@ "@babel/parser" "^7.14.5" "@babel/types" "^7.14.5" +"@babel/template@^7.18.10", "@babel/template@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.20.7.tgz#a15090c2839a83b02aa996c0b4994005841fd5a8" + integrity sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw== + dependencies: + "@babel/code-frame" "^7.18.6" + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + "@babel/traverse@7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.13.tgz#689f0e4b4c08587ad26622832632735fb8c4e0c0" @@ -1442,7 +1853,7 @@ globals "^11.1.0" lodash "^4.17.19" -"@babel/traverse@^7.0.0", "@babel/traverse@^7.1.0", "@babel/traverse@^7.12.1", "@babel/traverse@^7.13.0", "@babel/traverse@^7.13.15", "@babel/traverse@^7.14.0", "@babel/traverse@^7.14.2", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.0": +"@babel/traverse@^7.1.0", "@babel/traverse@^7.12.1", "@babel/traverse@^7.13.0", "@babel/traverse@^7.13.15", "@babel/traverse@^7.14.0", "@babel/traverse@^7.14.2", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.0": version "7.14.2" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.2.tgz#9201a8d912723a831c2679c7ebbf2fe1416d765b" integrity sha512-TsdRgvBFHMyHOOzcP9S6QU0QQtjxlRpEYOy3mcCO5RgmC305ki42aSAmfZEMSSYBla2oZ9BMqYlncBaKmD/7iA== @@ -1471,6 +1882,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.20.10", "@babel/traverse@^7.20.12", "@babel/traverse@^7.20.7": + version "7.20.12" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.20.12.tgz#7f0f787b3a67ca4475adef1f56cb94f6abd4a4b5" + integrity sha512-MsIbFN0u+raeja38qboyF8TIT7K0BFzz/Yd/77ta4MsUsmP2RAnidIlwq7d5HFQrH/OZJecGV6B71C4zAgpoSQ== + dependencies: + "@babel/code-frame" "^7.18.6" + "@babel/generator" "^7.20.7" + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-function-name" "^7.19.0" + "@babel/helper-hoist-variables" "^7.18.6" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + debug "^4.1.0" + globals "^11.1.0" + "@babel/types@7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.13.tgz#8be1aa8f2c876da11a9cf650c0ecf656913ad611" @@ -1496,6 +1923,15 @@ "@babel/helper-validator-identifier" "^7.14.5" to-fast-properties "^2.0.0" +"@babel/types@^7.18.6", "@babel/types@^7.19.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.7": + version "7.20.7" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.20.7.tgz#54ec75e252318423fc07fb644dc6a58a64c09b7f" + integrity sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg== + dependencies: + "@babel/helper-string-parser" "^7.19.4" + "@babel/helper-validator-identifier" "^7.19.1" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -1534,59 +1970,6 @@ resolved "https://registry.yarnpkg.com/@ctrl/tinycolor/-/tinycolor-3.4.0.tgz#c3c5ae543c897caa9c2a68630bed355be5f9990f" integrity sha512-JZButFdZ1+/xAfpguQHoabIXkcqRRKpMrWKBkpEZZyxfY9C1DpADFB8PEqGSTeFr135SaTRfKqGKx5xSCLI7ZQ== -"@cypress/listr-verbose-renderer@^0.4.1": - version "0.4.1" - resolved "https://registry.npmjs.org/@cypress/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz#a77492f4b11dcc7c446a34b3e28721afd33c642a" - integrity sha1-p3SS9LEdzHxEajSz4ochr9M8ZCo= - dependencies: - chalk "^1.1.3" - cli-cursor "^1.0.2" - date-fns "^1.27.2" - figures "^1.7.0" - -"@cypress/request@^2.88.5": - version "2.88.5" - resolved "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz#8d7ecd17b53a849cfd5ab06d5abe7d84976375d7" - integrity sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - har-validator "~5.1.3" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - oauth-sign "~0.9.0" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.5.0" - tunnel-agent "^0.6.0" - uuid "^3.3.2" - -"@cypress/webpack-preprocessor@5.8.0": - version "5.8.0" - resolved "https://registry.npmjs.org/@cypress/webpack-preprocessor/-/webpack-preprocessor-5.8.0.tgz#ae89775c0833d6d3c00fe9349ded270043cc6843" - integrity sha512-8R0EIWTPsOQQx1yGuQf6QFkaPxH3WtLbmIezeC4nWs8ITBTRASTTIhiXtbrntVphqFQX0Hm0eCiYEEwzBgVbog== - dependencies: - bluebird "^3.7.1" - debug "4.3.2" - lodash "^4.17.20" - -"@cypress/xvfb@^1.2.4": - version "1.2.4" - resolved "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz#2daf42e8275b39f4aa53c14214e557bd14e7748a" - integrity sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q== - dependencies: - debug "^3.1.0" - lodash.once "^4.1.1" - "@data-ui/shared@^0.0.84": version "0.0.84" resolved "https://registry.yarnpkg.com/@data-ui/shared/-/shared-0.0.84.tgz#42bd025d677f9be2beada3e1a84a53d33ac0eb10" @@ -1632,22 +2015,6 @@ d3-array "^1.2.0" prop-types "^15.5.10" -"@emotion/cache@^11.4.0", "@emotion/cache@^11.6.0": - version "11.6.0" - resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.6.0.tgz#65fbdbbe4382f1991d8b20853c38e63ecccec9a1" - integrity sha512-ElbsWY1KMwEowkv42vGo0UPuLgtPYfIs9BxxVrmvsaJVvktknsHYYlx5NQ5g6zLDcOTyamlDc7FkRg2TAcQDKQ== - dependencies: - "@emotion/memoize" "^0.7.4" - "@emotion/sheet" "^1.1.0" - "@emotion/utils" "^1.0.0" - "@emotion/weak-memoize" "^0.2.5" - stylis "^4.0.10" - -"@emotion/hash@^0.8.0": - version "0.8.0" - resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" - integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== - "@emotion/is-prop-valid@^0.8.8": version "0.8.8" resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-0.8.8.tgz#db28b1c4368a259b60a97311d6a952d4fd01ac1a" @@ -1660,60 +2027,16 @@ resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.4.tgz#19bf0f5af19149111c40d98bb0cf82119f5d9eeb" integrity sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw== -"@emotion/memoize@^0.7.4": - version "0.7.5" - resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.5.tgz#2c40f81449a4e554e9fc6396910ed4843ec2be50" - integrity sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ== - -"@emotion/react@^11.1.1": - version "11.7.0" - resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.7.0.tgz#b179da970ac0e8415de3ac165deadf8d9c4bf89f" - integrity sha512-WL93hf9+/2s3cA1JVJlz8+Uy6p6QWukqQFOm2OZO5ki51hfucHMOmbSjiyC3t2Y4RI8XUmBoepoc/24ny/VBbA== - dependencies: - "@babel/runtime" "^7.13.10" - "@emotion/cache" "^11.6.0" - "@emotion/serialize" "^1.0.2" - "@emotion/sheet" "^1.1.0" - "@emotion/utils" "^1.0.0" - "@emotion/weak-memoize" "^0.2.5" - hoist-non-react-statics "^3.3.1" - -"@emotion/serialize@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.0.2.tgz#77cb21a0571c9f68eb66087754a65fa97bfcd965" - integrity sha512-95MgNJ9+/ajxU7QIAruiOAdYNjxZX7G2mhgrtDWswA21VviYIRP1R5QilZ/bDY42xiKsaktP4egJb3QdYQZi1A== - dependencies: - "@emotion/hash" "^0.8.0" - "@emotion/memoize" "^0.7.4" - "@emotion/unitless" "^0.7.5" - "@emotion/utils" "^1.0.0" - csstype "^3.0.2" - -"@emotion/sheet@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.1.0.tgz#56d99c41f0a1cda2726a05aa6a20afd4c63e58d2" - integrity sha512-u0AX4aSo25sMAygCuQTzS+HsImZFuS8llY8O7b9MDRzbJM0kVJlAz6KNDqcG7pOuQZJmj/8X/rAW+66kMnMW+g== - "@emotion/stylis@^0.8.4": version "0.8.5" resolved "https://registry.yarnpkg.com/@emotion/stylis/-/stylis-0.8.5.tgz#deacb389bd6ee77d1e7fcaccce9e16c5c7e78e04" integrity sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ== -"@emotion/unitless@^0.7.4", "@emotion/unitless@^0.7.5": +"@emotion/unitless@^0.7.4": version "0.7.5" resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.7.5.tgz#77211291c1900a700b8a78cfafda3160d76949ed" integrity sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg== -"@emotion/utils@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.0.0.tgz#abe06a83160b10570816c913990245813a2fd6af" - integrity sha512-mQC2b3XLDs6QCW+pDQDiyO/EdGZYOygE8s5N5rrzjSI4M3IejPE/JPndCBwRT9z982aqQNi6beWs1UeayrQxxA== - -"@emotion/weak-memoize@^0.2.5": - version "0.2.5" - resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz#8eed982e2ee6f7f4e44c253e12962980791efd46" - integrity sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA== - "@endemolshinegroup/cosmiconfig-typescript-loader@3.0.2": version "3.0.2" resolved "https://registry.yarnpkg.com/@endemolshinegroup/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-3.0.2.tgz#eea4635828dde372838b0909693ebd9aafeec22d" @@ -1809,17 +2132,17 @@ tslib "~2.0.1" "@graphql-codegen/near-operation-file-preset@^1.17.13": - version "1.18.1" - resolved "https://registry.yarnpkg.com/@graphql-codegen/near-operation-file-preset/-/near-operation-file-preset-1.18.1.tgz#79ea5bb3db85498121ba30d9311341ea6021acea" - integrity sha512-/QowrCantXEymd7sAK/ZJlmHXboq5zbo0Y2vraEBkpkN3CAqw56lfXKJ5zLk0nuKLGtMD58c/pV9yqtgLI8Syw== + version "1.18.6" + resolved "https://registry.yarnpkg.com/@graphql-codegen/near-operation-file-preset/-/near-operation-file-preset-1.18.6.tgz#2378ac75feaeaa1cfd2146bd84bf839b1fe20d9d" + integrity sha512-MnNS0tWJlENvbqqoKc56Kv5yz9hsSDDKD3skXtBXiuAq4VzMoCC6nfpDdJXiYgbpLiVuuVQhbFFalVo9zqTEuA== dependencies: "@graphql-codegen/add" "^2.0.2" - "@graphql-codegen/plugin-helpers" "^1.18.7" - "@graphql-codegen/visitor-plugin-common" "1.21.0" + "@graphql-codegen/plugin-helpers" "^1.18.8" + "@graphql-codegen/visitor-plugin-common" "1.22.0" parse-filepath "^1.0.2" - tslib "~2.2.0" + tslib "~2.3.0" -"@graphql-codegen/plugin-helpers@^1.18.2", "@graphql-codegen/plugin-helpers@^1.18.7": +"@graphql-codegen/plugin-helpers@^1.18.2": version "1.18.7" resolved "https://registry.yarnpkg.com/@graphql-codegen/plugin-helpers/-/plugin-helpers-1.18.7.tgz#465af3e5b02de89e49ddc76ad2546b880fe240f2" integrity sha512-8ICOrXlsvyL1dpVz8C9b7H31d4DJpDd75WfjMn6Xjqz81Ah8xDn1Bi+7YXRCCILCBmvI94k6fi8qpsIVhFBBjQ== @@ -1830,15 +2153,16 @@ lodash "~4.17.0" tslib "~2.2.0" -"@graphql-codegen/typescript-document-nodes@1.17.9": - version "1.17.9" - resolved "https://registry.yarnpkg.com/@graphql-codegen/typescript-document-nodes/-/typescript-document-nodes-1.17.9.tgz#ac4f35b216617d6cf8b4056adc9c36f3b1b63f7c" - integrity sha512-4sSztahbc0YzoWbSlu/LA99f+gsXNYmXYucy7oEGdTsIDDAsaXoL/ZA7qzu+CwugPEOrP+3ZQiJqg+Btt6sfNQ== +"@graphql-codegen/plugin-helpers@^1.18.8": + version "1.18.8" + resolved "https://registry.yarnpkg.com/@graphql-codegen/plugin-helpers/-/plugin-helpers-1.18.8.tgz#39aac745b9e22e28c781cc07cf74836896a3a905" + integrity sha512-mb4I9j9lMGqvGggYuZ0CV+Hme08nar68xkpPbAVotg/ZBmlhZIok/HqW2BcMQi7Rj+Il5HQMeQ1wQ1M7sv/TlQ== dependencies: - "@graphql-codegen/plugin-helpers" "^1.18.2" - "@graphql-codegen/visitor-plugin-common" "^1.17.20" - auto-bind "~4.0.0" - tslib "~2.0.1" + "@graphql-tools/utils" "^7.9.1" + common-tags "1.8.0" + import-from "4.0.0" + lodash "~4.17.0" + tslib "~2.3.0" "@graphql-codegen/typescript-operations@1.17.13": version "1.17.13" @@ -1863,32 +2187,22 @@ pascal-case "^3.1.1" tslib "~2.0.1" -"@graphql-codegen/typescript@1.20.0": - version "1.20.0" - resolved "https://registry.yarnpkg.com/@graphql-codegen/typescript/-/typescript-1.20.0.tgz#f9a17b869e5691276965a56c7a1efe4eb938b6e7" - integrity sha512-7xW+n0USNpr6iZ4Et17ZbPzBLNe/LrSgrQ6V/8Mlgp1reQWAZtoVw13Oq4GnxHCzAYio8nFindLl+emW9ZBeew== - dependencies: - "@graphql-codegen/plugin-helpers" "^1.18.2" - "@graphql-codegen/visitor-plugin-common" "^1.18.0" - auto-bind "~4.0.0" - tslib "~2.0.1" - "@graphql-codegen/typescript@^1.18.1": - version "1.22.1" - resolved "https://registry.yarnpkg.com/@graphql-codegen/typescript/-/typescript-1.22.1.tgz#9a2e215d4cf095d942c1671db4bad78b619b11ce" - integrity sha512-3f/siciXrhhMdcs9qcxnwWXETsAhZNNiUOlr6IUEm82kVx5xvFuxc0KZZE88w3iEVJXE7xYo1oWmrttvkQP4Aw== + version "1.23.0" + resolved "https://registry.yarnpkg.com/@graphql-codegen/typescript/-/typescript-1.23.0.tgz#48a5372bcbe81a442c71c1bb032c312c6586a59a" + integrity sha512-ZfFgk5mGfuOy4kEpy+dcuvJMphigMfJ4AkiP1qWmWFufDW3Sg2yayTSNmzeFdcXMrWGgfNW2dKtuuTmbmQhS5g== dependencies: - "@graphql-codegen/plugin-helpers" "^1.18.7" - "@graphql-codegen/visitor-plugin-common" "1.21.0" + "@graphql-codegen/plugin-helpers" "^1.18.8" + "@graphql-codegen/visitor-plugin-common" "1.22.0" auto-bind "~4.0.0" - tslib "~2.2.0" + tslib "~2.3.0" -"@graphql-codegen/visitor-plugin-common@1.21.0", "@graphql-codegen/visitor-plugin-common@^1.17.20", "@graphql-codegen/visitor-plugin-common@^1.17.22", "@graphql-codegen/visitor-plugin-common@^1.18.0": - version "1.21.0" - resolved "https://registry.yarnpkg.com/@graphql-codegen/visitor-plugin-common/-/visitor-plugin-common-1.21.0.tgz#1cb59a8ce9a9d6486f859a254645e162c6736cfb" - integrity sha512-gw6mUCOpKwJ4aR+wnUXureQi4dV6jezEiXvX0CEZdpqBIGAJ4G8h7CKyGW66lBzvaoLCB7siOF86UJO3dw5ctg== +"@graphql-codegen/visitor-plugin-common@1.22.0", "@graphql-codegen/visitor-plugin-common@^1.17.20", "@graphql-codegen/visitor-plugin-common@^1.17.22": + version "1.22.0" + resolved "https://registry.yarnpkg.com/@graphql-codegen/visitor-plugin-common/-/visitor-plugin-common-1.22.0.tgz#75fc8b580143bccbec411eb92d5fef715ed22e42" + integrity sha512-2afJGb6d8iuZl9KizYsexPwraEKO1lAvt5eVHNM5Xew4vwz/AUHeqDR2uOeQgVV+27EzjjzSDd47IEdH0dLC2w== dependencies: - "@graphql-codegen/plugin-helpers" "^1.18.7" + "@graphql-codegen/plugin-helpers" "^1.18.8" "@graphql-tools/optimize" "^1.0.1" "@graphql-tools/relay-operation-optimizer" "^6.3.0" array.prototype.flatmap "^1.2.4" @@ -1897,7 +2211,7 @@ dependency-graph "^0.11.0" graphql-tag "^2.11.0" parse-filepath "^1.0.2" - tslib "~2.2.0" + tslib "~2.3.0" "@graphql-tools/apollo-engine-loader@^6": version "6.2.5" @@ -2020,11 +2334,11 @@ tslib "~2.2.0" "@graphql-tools/optimize@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@graphql-tools/optimize/-/optimize-1.0.1.tgz#9933fffc5a3c63f95102b1cb6076fb16ac7bb22d" - integrity sha512-cRlUNsbErYoBtzzS6zXahXeTBZGPVlPHXCpnEZ0XiK/KY/sQL96cyzak0fM/Gk6qEI9/l32MYEICjasiBQrl5w== + version "1.3.1" + resolved "https://registry.yarnpkg.com/@graphql-tools/optimize/-/optimize-1.3.1.tgz#29407991478dbbedc3e7deb8c44f46acb4e9278b" + integrity sha512-5j5CZSRGWVobt4bgRRg7zhjPiSimk+/zIuColih8E8DxuFOaJ+t0qu7eZS5KXWBkjcd4BPNuhUPpNlEmHPqVRQ== dependencies: - tslib "~2.0.1" + tslib "^2.4.0" "@graphql-tools/prisma-loader@^6": version "6.3.0" @@ -2054,13 +2368,13 @@ yaml-ast-parser "^0.0.43" "@graphql-tools/relay-operation-optimizer@^6.3.0": - version "6.3.0" - resolved "https://registry.yarnpkg.com/@graphql-tools/relay-operation-optimizer/-/relay-operation-optimizer-6.3.0.tgz#f8c7f6c8aa4a9cf50ab151fbc5db4f4282a79532" - integrity sha512-Or3UgRvkY9Fq1AAx7q38oPqFmTepLz7kp6wDHKyR0ceG7AvHv5En22R12mAeISInbhff4Rpwgf6cE8zHRu6bCw== + version "6.5.14" + resolved "https://registry.yarnpkg.com/@graphql-tools/relay-operation-optimizer/-/relay-operation-optimizer-6.5.14.tgz#e3d61892910c982c13ea8c2d9780a0cf95e7dd12" + integrity sha512-RAy1fMfXig9X3gIkYnfEmv0mh20vZuAgWDq+zf1MrrsCAP364B+DKrBjLwn3D+4e0PMTlqwmqR0JB5t1VtZn2w== dependencies: - "@graphql-tools/utils" "^7.1.0" - relay-compiler "10.1.0" - tslib "~2.0.1" + "@ardatan/relay-compiler" "12.0.0" + "@graphql-tools/utils" "9.1.3" + tslib "^2.4.0" "@graphql-tools/schema@^7.0.0", "@graphql-tools/schema@^7.1.5": version "7.1.5" @@ -2096,6 +2410,13 @@ valid-url "1.0.9" ws "7.4.5" +"@graphql-tools/utils@9.1.3": + version "9.1.3" + resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-9.1.3.tgz#861f87057b313726136fa6ddfbd2380eae906599" + integrity sha512-bbJyKhs6awp1/OmP+WKA1GOyu9UbgZGkhIj5srmiMGLHohEOKMjW784Sk0BZil1w2x95UPu0WHw6/d/HVCACCg== + dependencies: + tslib "^2.4.0" + "@graphql-tools/utils@^6": version "6.2.4" resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-6.2.4.tgz#38a2314d2e5e229ad4f78cca44e1199e18d55856" @@ -2105,7 +2426,7 @@ camel-case "4.1.1" tslib "~2.0.1" -"@graphql-tools/utils@^7.0.0", "@graphql-tools/utils@^7.1.0", "@graphql-tools/utils@^7.1.2", "@graphql-tools/utils@^7.5.0", "@graphql-tools/utils@^7.7.0", "@graphql-tools/utils@^7.7.1", "@graphql-tools/utils@^7.8.1", "@graphql-tools/utils@^7.9.0", "@graphql-tools/utils@^7.9.1": +"@graphql-tools/utils@^7.0.0", "@graphql-tools/utils@^7.1.2", "@graphql-tools/utils@^7.5.0", "@graphql-tools/utils@^7.7.0", "@graphql-tools/utils@^7.7.1", "@graphql-tools/utils@^7.8.1", "@graphql-tools/utils@^7.9.0", "@graphql-tools/utils@^7.9.1": version "7.10.0" resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-7.10.0.tgz#07a4cb5d1bec1ff1dc1d47a935919ee6abd38699" integrity sha512-d334r6bo9mxdSqZW6zWboEnnOOFRrAPVQJ7LkU8/6grglrbcu6WhwCLzHb90E94JI3TD3ricC3YGbUqIi9Xg0w== @@ -2371,6 +2692,46 @@ "@types/yargs" "^15.0.0" chalk "^4.0.0" +"@jridgewell/gen-mapping@^0.1.0": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996" + integrity sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w== + dependencies: + "@jridgewell/set-array" "^1.0.0" + "@jridgewell/sourcemap-codec" "^1.4.10" + +"@jridgewell/gen-mapping@^0.3.2": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" + integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A== + dependencies: + "@jridgewell/set-array" "^1.0.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.9" + +"@jridgewell/resolve-uri@3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" + integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== + +"@jridgewell/set-array@^1.0.0", "@jridgewell/set-array@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== + +"@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": + version "1.4.14" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" + integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== + +"@jridgewell/trace-mapping@^0.3.9": + version "0.3.17" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" + integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== + dependencies: + "@jridgewell/resolve-uri" "3.1.0" + "@jridgewell/sourcemap-codec" "1.4.14" + "@mapbox/rehype-prism@0.6.0": version "0.6.0" resolved "https://registry.yarnpkg.com/@mapbox/rehype-prism/-/rehype-prism-0.6.0.tgz#3d8a860870951d4354257d0ba908d11545bd5ed5" @@ -2534,14 +2895,6 @@ dependencies: any-observable "^0.3.0" -"@segment/loosely-validate-event@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@segment/loosely-validate-event/-/loosely-validate-event-2.0.0.tgz#87dfc979e5b4e7b82c5f1d8b722dfd5d77644681" - integrity sha512-ZMCSfztDBqwotkl848ODgVcAmN4OItEWDCkshcKz0/W6gGSQayuuCtWV/MlodFivAZD793d6UgANd6wCXUfrIw== - dependencies: - component-type "^1.2.1" - join-component "^1.1.0" - "@sideway/address@^4.1.0": version "4.1.2" resolved "https://registry.npmjs.org/@sideway/address/-/address-4.1.2.tgz#811b84333a335739d3969cfc434736268170cad1" @@ -2753,13 +3106,6 @@ "@babel/runtime" "^7.12.5" "@testing-library/dom" "^7.28.1" -"@testing-library/user-event@^12.6.0": - version "12.8.3" - resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-12.8.3.tgz#1aa3ed4b9f79340a1e1836bc7f57c501e838704a" - integrity sha512-IR0iWbFkgd56Bu5ZI/ej8yQwrkCv8Qydx6RzwbKz9faXazR/+5tvYKsZQgyXJiwgpcva127YO6JcWy7YlCfofQ== - dependencies: - "@babel/runtime" "^7.12.5" - "@tommoor/remove-markdown@^0.3.2": version "0.3.2" resolved "https://registry.yarnpkg.com/@tommoor/remove-markdown/-/remove-markdown-0.3.2.tgz#5288ddd0e26b6b173e76ebb31c94653b0dcff45d" @@ -2966,7 +3312,7 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/jest@*", "@types/jest@^26.0.19": +"@types/jest@*": version "26.0.23" resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.23.tgz#a1b7eab3c503b80451d019efb588ec63522ee4e7" integrity sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA== @@ -2989,11 +3335,6 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== -"@types/json-schema@^7.0.0": - version "7.0.9" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.9.tgz#97edc9037ea0c38585320b28964dde3b39e4660d" - integrity sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ== - "@types/json-stable-stringify@^1.0.32": version "1.0.32" resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.32.tgz#121f6917c4389db3923640b2e68de5fa64dda88e" @@ -3038,11 +3379,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.13.tgz#e743bae112bd779ac9650f907197dd2caa7f0364" integrity sha512-1x8W5OpxPq+T85OUsHRP6BqXeosKmeXRtjoF39STcdf/UWLqUsoehstZKOi0CunhVqHG17AyZgpj20eRVooK6A== -"@types/node@^14.14.31": - version "14.17.5" - resolved "https://registry.npmjs.org/@types/node/-/node-14.17.5.tgz#b59daf6a7ffa461b5648456ca59050ba8e40ed54" - integrity sha512-bjqH2cX/O33jXT/UmReo2pM7DIJREPMnarixbQ57DOOzzFaI6D2+IcwaJQaJpv0M1E9TIhPCYVxrkcityLjlqA== - "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" @@ -3104,13 +3440,6 @@ "@types/history" "*" "@types/react" "*" -"@types/react-transition-group@^4.4.0": - version "4.4.4" - resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.4.tgz#acd4cceaa2be6b757db61ed7b432e103242d163e" - integrity sha512-7gAPz7anVK5xzbeQW9wFBDg7G++aPLAFY0QaSMOou9rJZpbuI58WAuJrgu+qR92l61grlnCUe7AFX8KGahAgug== - dependencies: - "@types/react" "*" - "@types/react@*", "@types/react@^17.0.0": version "17.0.8" resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.8.tgz#fe76e3ba0fbb5602704110fd1e3035cf394778e3" @@ -3132,16 +3461,6 @@ resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.1.tgz#18845205e86ff0038517aab7a18a62a6b9f71275" integrity sha512-EaCxbanVeyxDRTQBkdLb3Bvl/HK7PBK6UJjsSixB0iHKoWxE5uu2Q/DgtpOhPIojN0Zl1whvOd7PoHs2P0s5eA== -"@types/sinonjs__fake-timers@^6.0.2": - version "6.0.3" - resolved "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08" - integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g== - -"@types/sizzle@^2.3.2": - version "2.3.3" - resolved "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" - integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== - "@types/source-list-map@*": version "0.1.2" resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9" @@ -3362,39 +3681,6 @@ "@uiw/react-markdown-preview" "3.0.6" rehype "11.0.0" -"@visx/drag@^1.7.0": - version "1.7.4" - resolved "https://registry.yarnpkg.com/@visx/drag/-/drag-1.7.4.tgz#263b487ee14bcec66eb424c04f07bf81f4608606" - integrity sha512-lKih9Vlrt/4DsFM1Km1xGaWViXAZ3jRxsuNpUtCfcdrD0LN9zncDg4/P+gPuFN8eVXPoJb1x5bfhIYnt5hbvlQ== - dependencies: - "@types/react" "*" - "@visx/event" "1.7.0" - prop-types "^15.5.10" - -"@visx/event@1.7.0": - version "1.7.0" - resolved "https://registry.yarnpkg.com/@visx/event/-/event-1.7.0.tgz#76a1e726dfb72f33dffb957fd7e0c49488f426fe" - integrity sha512-RbAoKxvy+ildX2dVXC9/ZX94lQXPwjKgtO9jy7COc15knG4zmzsMCDYDC3uLd0+jE2o/+gSaZ/9r52p6zG5+IQ== - dependencies: - "@types/react" "*" - "@visx/point" "1.7.0" - -"@visx/point@1.7.0": - version "1.7.0" - resolved "https://registry.yarnpkg.com/@visx/point/-/point-1.7.0.tgz#1df3c3425eae464f498473bcdda2fcae05c8ecbe" - integrity sha512-oaoY/HXYHhmpkkeKI4rBPmFtjHWtxSrIhZCVm1ipPoyQp3voJ8L6JD5eUIVmmaUCdUGUGwL1lFLnJiQ2p1Vlwg== - -"@visx/responsive@^1.7.0": - version "1.10.1" - resolved "https://registry.yarnpkg.com/@visx/responsive/-/responsive-1.10.1.tgz#87c6ec030e6817aae90a588a155ea86159b098dd" - integrity sha512-7FT2BBmWFkFFqynI9C1NYfVOKT1FsNOm6MwWMqXKA7TMomdBW0wdtQNB1bHvwJvWurM/sNqxcQ/CBED6t9xujQ== - dependencies: - "@types/lodash" "^4.14.146" - "@types/react" "*" - lodash "^4.17.10" - prop-types "^15.6.1" - resize-observer-polyfill "1.5.1" - "@vx/axis@^0.0.175": version "0.0.175" resolved "https://registry.yarnpkg.com/@vx/axis/-/axis-0.0.175.tgz#312c07b81e8b043876436cab3bd8e90f30a1f7ec" @@ -3481,14 +3767,6 @@ classnames "^2.2.5" prop-types "^15.5.7" -"@vx/gradient@^0.0.199": - version "0.0.199" - resolved "https://registry.yarnpkg.com/@vx/gradient/-/gradient-0.0.199.tgz#f30fd62281281c39127438e09fc4a793066b62bf" - integrity sha512-DI4PBH5+eDquTXS1vVEv6VtTZlDmj2g3DRPuDq7BL2EXTSI8tdyEdxkDHTWeO2XHLaltVSR9YTykFbfp3VBiEw== - dependencies: - "@types/react" "*" - prop-types "^15.5.7" - "@vx/grid@^0.0.180": version "0.0.180" resolved "https://registry.yarnpkg.com/@vx/grid/-/grid-0.0.180.tgz#31f2f73c9055d0ab2bf38f03a0c7dc47b9c12327" @@ -3500,20 +3778,6 @@ classnames "^2.2.5" prop-types "^15.6.2" -"@vx/grid@^0.0.199": - version "0.0.199" - resolved "https://registry.yarnpkg.com/@vx/grid/-/grid-0.0.199.tgz#df583770bda8488faadb6f7a31b09c0538436f82" - integrity sha512-Dh6HiFLJ4Pwmk9T6znZRASM4Jnjug95zfZIV3Kt3S8RkD8L2gpkCmCtdTA77yBbBFUMzH1/cXvvqqsFgapeNmQ== - dependencies: - "@types/classnames" "^2.2.9" - "@types/react" "*" - "@vx/group" "0.0.199" - "@vx/point" "0.0.199" - "@vx/scale" "0.0.199" - "@vx/shape" "0.0.199" - classnames "^2.2.5" - prop-types "^15.6.2" - "@vx/group@0.0.165", "@vx/group@^0.0.165": version "0.0.165" resolved "https://registry.yarnpkg.com/@vx/group/-/group-0.0.165.tgz#2342523225de94859b5be49c3072dc6bb6795e78" @@ -3926,13 +4190,6 @@ dependencies: tslib "^2.1.0" -"@wry/equality@^0.1.2": - version "0.1.11" - resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.1.11.tgz#35cb156e4a96695aa81a9ecc4d03787bc17f1790" - integrity sha512-mwEVBDUVODlsQQ5dfuLUS5/Tf7jqUKyhKYHmVi4fPB6bDMOfWvUPJmKgS1Z7Za/sOI3vzWt4+O7yCiL/70MogA== - dependencies: - tslib "^1.9.3" - "@wry/equality@^0.4.0": version "0.4.0" resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.4.0.tgz#474491869a8d0590f4a33fd2a4850a77a0f63408" @@ -4073,20 +4330,6 @@ alphanum-sort@^1.0.0: resolved "https://registry.yarnpkg.com/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3" integrity sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM= -analytics-node@^3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/analytics-node/-/analytics-node-3.5.0.tgz#b33ff92195f006b20f1c4e28af86d975c98e4636" - integrity sha512-XgQq6ejZHCehUSnZS4V7QJPLIP7S9OAWwQDYl4WTLtsRvc5fCxIwzK/yihzmIW51v9PnyBmrl9dMcqvwfOE8WA== - dependencies: - "@segment/loosely-validate-event" "^2.0.0" - axios "^0.21.1" - axios-retry "^3.0.2" - lodash.isstring "^4.0.1" - md5 "^2.2.1" - ms "^2.0.0" - remove-trailing-slash "^0.1.0" - uuid "^3.2.1" - analytics-utils@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/analytics-utils/-/analytics-utils-0.4.4.tgz#0fc032c8d3dd8a37f6275ea47f678958204d5bf3" @@ -4226,76 +4469,24 @@ any-observable@^0.3.0: anymatch@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb" - integrity sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw== - dependencies: - micromatch "^3.1.4" - normalize-path "^2.1.1" - -anymatch@^3.0.0, anymatch@^3.0.3, anymatch@~3.1.1: - version "3.1.2" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" - integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== - dependencies: - normalize-path "^3.0.0" - picomatch "^2.0.4" - -apollo-link-error@^1.1.13: - version "1.1.13" - resolved "https://registry.yarnpkg.com/apollo-link-error/-/apollo-link-error-1.1.13.tgz#c1a1bb876ffe380802c8df0506a32c33aad284cd" - integrity sha512-jAZOOahJU6bwSqb2ZyskEK1XdgUY9nkmeclCrW7Gddh1uasHVqmoYc4CKdb0/H0Y1J9lvaXKle2Wsw/Zx1AyUg== - dependencies: - apollo-link "^1.2.14" - apollo-link-http-common "^0.2.16" - tslib "^1.9.3" - -apollo-link-http-common@^0.2.16: - version "0.2.16" - resolved "https://registry.yarnpkg.com/apollo-link-http-common/-/apollo-link-http-common-0.2.16.tgz#756749dafc732792c8ca0923f9a40564b7c59ecc" - integrity sha512-2tIhOIrnaF4UbQHf7kjeQA/EmSorB7+HyJIIrUjJOKBgnXwuexi8aMecRlqTIDWcyVXCeqLhUnztMa6bOH/jTg== - dependencies: - apollo-link "^1.2.14" - ts-invariant "^0.4.0" - tslib "^1.9.3" - -apollo-link-http@^1.5.17: - version "1.5.17" - resolved "https://registry.yarnpkg.com/apollo-link-http/-/apollo-link-http-1.5.17.tgz#499e9f1711bf694497f02c51af12d82de5d8d8ba" - integrity sha512-uWcqAotbwDEU/9+Dm9e1/clO7hTB2kQ/94JYcGouBVLjoKmTeJTUPQKcJGpPwUjZcSqgYicbFqQSoJIW0yrFvg== - dependencies: - apollo-link "^1.2.14" - apollo-link-http-common "^0.2.16" - tslib "^1.9.3" - -apollo-link@^1.2.14: - version "1.2.14" - resolved "https://registry.yarnpkg.com/apollo-link/-/apollo-link-1.2.14.tgz#3feda4b47f9ebba7f4160bef8b977ba725b684d9" - integrity sha512-p67CMEFP7kOG1JZ0ZkYZwRDa369w5PIjtMjvrQd/HnIV8FRsHRqLqK+oAZQnFa1DDdZtOtHTi+aMIW6EatC2jg== + integrity sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw== dependencies: - apollo-utilities "^1.3.0" - ts-invariant "^0.4.0" - tslib "^1.9.3" - zen-observable-ts "^0.8.21" + micromatch "^3.1.4" + normalize-path "^2.1.1" -apollo-utilities@^1.3.0: - version "1.3.4" - resolved "https://registry.yarnpkg.com/apollo-utilities/-/apollo-utilities-1.3.4.tgz#6129e438e8be201b6c55b0f13ce49d2c7175c9cf" - integrity sha512-pk2hiWrCXMAy2fRPwEyhvka+mqwzeP60Jr1tRYi5xru+3ko94HI9o6lK0CT33/w4RDlxWchmdhDCrvdr+pHCig== +anymatch@^3.0.0, anymatch@^3.0.3, anymatch@~3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" + integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== dependencies: - "@wry/equality" "^0.1.2" - fast-json-stable-stringify "^2.0.0" - ts-invariant "^0.4.0" - tslib "^1.10.0" + normalize-path "^3.0.0" + picomatch "^2.0.4" aproba@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== -arch@^2.2.0: - version "2.2.0" - resolved "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" - integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== - arg@^4.1.0: version "4.1.3" resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" @@ -4490,11 +4681,6 @@ async@^2.6.2: dependencies: lodash "^4.17.14" -async@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" - integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -4543,13 +4729,6 @@ axe-core@^4.0.2: resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.2.1.tgz#2e50bcf10ee5b819014f6e342e41e45096239e34" integrity sha512-evY7DN8qSIbsW2H/TWQ1bX3sXN1d4MNb5Vb4n7BzPuCwRHdkZ1H2eNLuSh73EoQqkGKUtju2G2HCcjCfhvZIAA== -axios-retry@^3.0.2: - version "3.1.9" - resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.1.9.tgz#6c30fc9aeb4519aebaec758b90ef56fa03fe72e8" - integrity sha512-NFCoNIHq8lYkJa6ku4m+V1837TP6lCa7n79Iuf8/AqATAHYB0ISaAS1eyIenDOfHOLtym34W65Sjke2xjg2fsA== - dependencies: - is-retry-allowed "^1.1.0" - axios@^0.21.1: version "0.21.4" resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" @@ -4766,7 +4945,7 @@ babel-preset-current-node-syntax@^1.0.0: "@babel/plugin-syntax-optional-chaining" "^7.8.3" "@babel/plugin-syntax-top-level-await" "^7.8.3" -babel-preset-fbjs@^3.3.0: +babel-preset-fbjs@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/babel-preset-fbjs/-/babel-preset-fbjs-3.4.0.tgz#38a14e5a7a3b285a3f3a86552d650dca5cf6111c" integrity sha512-9ywCsCvo1ojrw0b+XYk7aFvTH6D9064t0RIL1rtMf3nsa02Xw41MS7sZw216Im35xj/UY0PDBQsa1brUDDF1Ow== @@ -4928,21 +5107,11 @@ blacklist@^1.1.4: resolved "https://registry.yarnpkg.com/blacklist/-/blacklist-1.1.4.tgz#b2dd09d6177625b2caa69835a37b28995fa9a2f2" integrity sha512-DWdfwimA1WQxVC69Vs1Fy525NbYwisMSCdYQmW9zyzOByz9OB/tQwrKZ3T3pbTkuFjnkJFlJuyiDjPiXL5kzew== -blob-util@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" - integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== - -bluebird@3.7.2, bluebird@^3.5.5, bluebird@^3.7.1, bluebird@^3.7.2: +bluebird@3.7.2, bluebird@^3.5.5: version "3.7.2" resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -bluebird@~2.2.2: - version "2.2.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.2.2.tgz#f1bf05abc887cf9a703886237c28610a43b1f114" - integrity sha1-8b8Fq8iHz5pwOIYjfChhCkOx8RQ= - bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9: version "4.12.0" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88" @@ -5111,6 +5280,16 @@ browserslist@^4.0.0, browserslist@^4.12.0, browserslist@^4.14.5, browserslist@^4 escalade "^3.1.1" node-releases "^1.1.71" +browserslist@^4.21.3: + version "4.21.4" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.4.tgz#e7496bbc67b9e39dd0f98565feccdcb0d4ff6987" + integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== + dependencies: + caniuse-lite "^1.0.30001400" + electron-to-chromium "^1.4.251" + node-releases "^2.0.6" + update-browserslist-db "^1.0.9" + bser@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" @@ -5123,11 +5302,6 @@ btoa@^1.2.1: resolved "https://registry.yarnpkg.com/btoa/-/btoa-1.2.1.tgz#01a9909f8b2c93f6bf680ba26131eb30f7fa3d73" integrity sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g== -buffer-crc32@~0.2.3: - version "0.2.13" - resolved "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" - integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= - buffer-equal-constant-time@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" @@ -5257,11 +5431,6 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" -cachedir@^2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz#0c75892a052198f0b21c7c1804d8331edfcae0e8" - integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw== - call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" @@ -5340,6 +5509,11 @@ caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001109, can resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001230.tgz#8135c57459854b2240b57a4a6786044bdc5a9f71" integrity sha512-5yBd5nWCBS+jWKTcHOzXwo5xzcj4ePE/yjtkZyUV1BTUmrBaA9MRGC+e7mxnqXSA90CmCA8L3eKLaSUkt099IQ== +caniuse-lite@^1.0.30001400: + version "1.0.30001442" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001442.tgz#40337f1cf3be7c637b061e2f78582dc1daec0614" + integrity sha512-239m03Pqy0hwxYPYR5JwOIxRJfLTWtle9FV8zosfV5pHg+/51uD4nxcUlM8+mWWGfwKtt8lJNHnD3cWw9VZ6ow== + capital-case@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669" @@ -5471,12 +5645,7 @@ chardet@^0.7.0: resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== -charenc@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" - integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc= - -check-more-types@2.24.0, check-more-types@^2.24.0: +check-more-types@2.24.0: version "2.24.0" resolved "https://registry.npmjs.org/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" integrity sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA= @@ -5540,11 +5709,6 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== -ci-info@^3.1.1: - version "3.2.0" - resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.2.0.tgz#2876cb948a498797b5236f0095bc057d0dca38b6" - integrity sha512-dVqRX7fLUm8J6FgHJ418XuIgDLZDkYcDFTeL6TA2gt5WlIZUQrrH6EZrNClwT/H0FateUsZkGIOPRrLbP+PR9A== - cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" @@ -5590,21 +5754,6 @@ clean-stack@^2.0.0: resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== -cli-color@~0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/cli-color/-/cli-color-0.2.3.tgz#0a25ceae5a6a1602be7f77d28563c36700274e88" - integrity sha1-CiXOrlpqFgK+f3fShWPDZwAnTog= - dependencies: - es5-ext "~0.9.2" - memoizee "~0.2.5" - -cli-cursor@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987" - integrity sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc= - dependencies: - restore-cursor "^1.0.1" - cli-cursor@^2.0.0, cli-cursor@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5" @@ -5619,16 +5768,6 @@ cli-cursor@^3.1.0: dependencies: restore-cursor "^3.1.0" -cli-table3@~0.6.0: - version "0.6.0" - resolved "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.0.tgz#b7b1bc65ca8e7b5cef9124e13dc2b21e2ce4faee" - integrity sha512-gnB85c3MGC7Nm9I/FkiasNBOKjOiO1RNuXXarQms37q4QMpWdlbBgD/VnOStA2faG1dpXMv31RFApjX1/QdgWQ== - dependencies: - object-assign "^4.1.0" - string-width "^4.2.0" - optionalDependencies: - colors "^1.1.2" - cli-truncate@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-0.2.1.tgz#9f15cfbb0705005369216c626ac7d05ab90dd574" @@ -5772,11 +5911,6 @@ colorette@^1.2.1, colorette@^1.2.2: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== -colors@^1.1.2: - version "1.4.0" - resolved "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" - integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== - combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -5799,11 +5933,6 @@ commander@^4.1.1: resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068" integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA== -commander@^5.1.0: - version "5.1.0" - resolved "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" - integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== - common-tags@1.8.0, common-tags@^1.8.0: version "1.8.0" resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937" @@ -5819,11 +5948,6 @@ component-emitter@^1.2.1: resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== -component-type@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9" - integrity sha1-ikeQFwAjjk/DIml3EjAibyS0Fak= - compose-function@3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/compose-function/-/compose-function-3.0.3.tgz#9ed675f13cc54501d30950a486ff6a7ba3ab185f" @@ -5861,7 +5985,7 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= -concat-stream@^1.5.0, concat-stream@^1.6.2: +concat-stream@^1.5.0: version "1.6.2" resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== @@ -6122,13 +6246,20 @@ cross-fetch@3.0.6: dependencies: node-fetch "2.6.1" -cross-fetch@3.1.4, cross-fetch@^3.0.4, cross-fetch@^3.0.6: +cross-fetch@3.1.4, cross-fetch@^3.0.6: version "3.1.4" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39" integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ== dependencies: node-fetch "2.6.1" +cross-fetch@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" + integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== + dependencies: + node-fetch "2.6.7" + cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -6149,11 +6280,6 @@ cross-spawn@^6.0.0: shebang-command "^1.2.0" which "^1.2.9" -crypt@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" - integrity sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs= - crypto-browserify@^3.11.0: version "3.12.0" resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec" @@ -6420,51 +6546,6 @@ cyclist@^1.0.1: resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= -cypress@7.3.0: - version "7.3.0" - resolved "https://registry.npmjs.org/cypress/-/cypress-7.3.0.tgz#17345b8d18681c120f033e7d8fd0f0271e9d0d51" - integrity sha512-aseRCH1tRVCrM6oEfja6fR/bo5l6e4SkHRRSATh27UeN4f/ANC8U7tGIulmrISJVy9xuOkOdbYKbUb2MNM+nrw== - dependencies: - "@cypress/listr-verbose-renderer" "^0.4.1" - "@cypress/request" "^2.88.5" - "@cypress/xvfb" "^1.2.4" - "@types/node" "^14.14.31" - "@types/sinonjs__fake-timers" "^6.0.2" - "@types/sizzle" "^2.3.2" - arch "^2.2.0" - blob-util "^2.0.2" - bluebird "^3.7.2" - cachedir "^2.3.0" - chalk "^4.1.0" - check-more-types "^2.24.0" - cli-table3 "~0.6.0" - commander "^5.1.0" - common-tags "^1.8.0" - dayjs "^1.10.4" - debug "4.3.2" - eventemitter2 "^6.4.3" - execa "4.1.0" - executable "^4.1.1" - extract-zip "^1.7.0" - fs-extra "^9.1.0" - getos "^3.2.1" - is-ci "^3.0.0" - is-installed-globally "~0.4.0" - lazy-ass "^1.6.0" - listr "^0.14.3" - lodash "^4.17.21" - log-symbols "^4.0.0" - minimist "^1.2.5" - ospath "^1.2.2" - pretty-bytes "^5.6.0" - ramda "~0.27.1" - request-progress "^3.0.0" - supports-color "^8.1.1" - tmp "~0.2.1" - untildify "^4.0.0" - url "^0.11.0" - yauzl "^2.10.0" - d3-array@2, d3-array@^2.3.0: version "2.12.1" resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.12.1.tgz#e20b41aafcdffdf5d50928004ececf815a465e81" @@ -6563,7 +6644,7 @@ d3-time-format@2: dependencies: d3-time "1" -"d3-time-format@2 - 3", d3-time-format@^3.0.0: +"d3-time-format@2 - 3": version "3.0.0" resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-3.0.0.tgz#df8056c83659e01f20ac5da5fdeae7c08d5f1bb6" integrity sha512-UXJh6EKsHBTjopVqZBhFysQcoXSv/5yLONZvkQ5Kk3qbwiUYkdX17Xa1PT6U1ZWXGGfB1ey5L8dKMlFq2DO0Ag== @@ -6636,11 +6717,6 @@ dayjs@1.x: resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.6.tgz#2e79a226314ec3ec904e3ee1dd5a4f5e5b1c7afb" integrity sha512-zZbY5giJAinCG+7AGaw0wIhNZ6J8AhWuSXKvuc1KAyMiRsvGQWqh4L+MomvhdAYjN+lqvVCMq1I41e3YHvXkyQ== -dayjs@^1.10.4: - version "1.10.6" - resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.10.6.tgz#288b2aa82f2d8418a6c9d4df5898c0737ad02a63" - integrity sha512-AztC/IOW4L1Q41A86phW5Thhcrco3xuAA+YX/BLpLWWjRcTj5TOt/QImBLmCKlrF7u7k47arTnOyL6GnbG8Hvw== - debounce@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.1.tgz#38881d8f4166a5c5848020c11827b834bcb3e0a5" @@ -6653,7 +6729,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4, debug@4.3.2, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1: +debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1: version "4.3.2" resolved "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -6674,7 +6750,7 @@ debug@4.3.1: dependencies: ms "2.1.2" -debug@^3.1.0, debug@^3.1.1, debug@^3.2.6, debug@^3.2.7: +debug@^3.1.1, debug@^3.2.6, debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -6948,14 +7024,6 @@ dom-converter@^0.2: dependencies: utila "~0.4" -dom-helpers@^5.0.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/dom-helpers/-/dom-helpers-5.2.1.tgz#d9400536b2bf8225ad98fe052e029451ac40e902" - integrity sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA== - dependencies: - "@babel/runtime" "^7.8.7" - csstype "^3.0.2" - dom-serializer@0: version "0.2.2" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51" @@ -7093,6 +7161,11 @@ electron-to-chromium@^1.3.564, electron-to-chromium@^1.3.723: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.739.tgz#f07756aa92cabd5a6eec6f491525a64fe62f98b9" integrity sha512-+LPJVRsN7hGZ9EIUUiWCpO7l4E3qBYHNadazlucBfsXBbccDFNKUBAgzE68FnkWGJPwD/AfKhSzL+G+Iqb8A4A== +electron-to-chromium@^1.4.251: + version "1.4.284" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592" + integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== + elegant-spinner@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz#db043521c95d7e303fd8f345bedc3349cfb0729e" @@ -7240,11 +7313,6 @@ es5-ext@^0.10.35, es5-ext@^0.10.50: es6-symbol "~3.1.3" next-tick "~1.0.0" -es5-ext@~0.9.2: - version "0.9.2" - resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.9.2.tgz#d2e309d1f223b0718648835acf5b8823a8061f8a" - integrity sha1-0uMJ0fIjsHGGSINaz1uII6gGH4o= - es6-iterator@2.0.3, es6-iterator@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" @@ -7592,13 +7660,6 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -event-emitter@~0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.2.2.tgz#c81e3724eb55407c5a0d5ee3299411f700f54291" - integrity sha1-yB43JOtVQHxaDV7jKZQR9wD1QpE= - dependencies: - es5-ext "~0.9.2" - event-stream@=3.3.4: version "3.3.4" resolved "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz#4ab4c9a0f5a54db9338b4c34d86bfce8f4b35571" @@ -7617,11 +7678,6 @@ event-target-shim@^5.0.0: resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== -eventemitter2@^6.4.3: - version "6.4.4" - resolved "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.4.tgz#aa96e8275c4dbeb017a5d0e03780c65612a1202b" - integrity sha512-HLU3NDY6wARrLCEwyGKRBvuWYyvW6mHYv72SJJAH3iJN3a6eVUvkjFkcxah1bcTgGVBBrFdIopBJPhCQFMLyXw== - eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -7657,21 +7713,6 @@ exec-sh@^0.3.2: resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.6.tgz#ff264f9e325519a60cb5e273692943483cca63bc" integrity sha512-nQn+hI3yp+oD0huYhKwvYI32+JFeq+XkNcD1GAo3Y/MjxsfVGmrrzrnzjWiNY6f+pUCP440fThsFh5gZrRAU/w== -execa@4.1.0, execa@^4.0.0: - version "4.1.0" - resolved "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" - integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== - dependencies: - cross-spawn "^7.0.0" - get-stream "^5.0.0" - human-signals "^1.1.1" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.0" - onetime "^5.1.0" - signal-exit "^3.0.2" - strip-final-newline "^2.0.0" - execa@5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz#4029b0007998a841fbd1032e5f4de86a3c1e3376" @@ -7700,17 +7741,20 @@ execa@^1.0.0: signal-exit "^3.0.0" strip-eof "^1.0.0" -executable@^4.1.1: - version "4.1.1" - resolved "https://registry.npmjs.org/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" - integrity sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg== +execa@^4.0.0: + version "4.1.0" + resolved "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" + integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== dependencies: - pify "^2.2.0" - -exit-hook@^1.0.0: - version "1.1.1" - resolved "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz#f05ca233b48c05d54fff07765df8507e95c02ff8" - integrity sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g= + cross-spawn "^7.0.0" + get-stream "^5.0.0" + human-signals "^1.1.1" + is-stream "^2.0.0" + merge-stream "^2.0.0" + npm-run-path "^4.0.0" + onetime "^5.1.0" + signal-exit "^3.0.2" + strip-final-newline "^2.0.0" exit@^0.1.2: version "0.1.2" @@ -7834,16 +7878,6 @@ extract-files@9.0.0, extract-files@^9.0.0: resolved "https://registry.yarnpkg.com/extract-files/-/extract-files-9.0.0.tgz#8a7744f2437f81f5ed3250ed9f1550de902fe54a" integrity sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ== -extract-zip@^1.7.0: - version "1.7.0" - resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" - integrity sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA== - dependencies: - concat-stream "^1.6.2" - debug "^2.6.9" - mkdirp "^0.5.4" - yauzl "^2.10.0" - extsprintf@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" @@ -7941,24 +7975,17 @@ fbjs-css-vars@^1.0.0: integrity sha512-b2XGFAFdWZWg0phtAWLHCk836A1Xann+I+Dgd3Gk64MHKZO44FfoD1KxyvbSh0qZsIoXQGGlVztIY+oitJPpRQ== fbjs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-3.0.0.tgz#0907067fb3f57a78f45d95f1eacffcacd623c165" - integrity sha512-dJd4PiDOFuhe7vk4F80Mba83Vr2QuK86FoxtgPmzBqEJahncp+13YCmfoa53KHCo6OnlXLG7eeMWPfB5CrpVKg== + version "3.0.4" + resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-3.0.4.tgz#e1871c6bd3083bac71ff2da868ad5067d37716c6" + integrity sha512-ucV0tDODnGV3JCnnkmoszb5lf4bNpzjv80K41wd4k798Etq+UYD0y0TIfalLjZoKgjive6/adkRnszwapiDgBQ== dependencies: - cross-fetch "^3.0.4" + cross-fetch "^3.1.5" fbjs-css-vars "^1.0.0" loose-envify "^1.0.0" object-assign "^4.1.0" promise "^7.1.1" setimmediate "^1.0.5" - ua-parser-js "^0.7.18" - -fd-slicer@~1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" - integrity sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4= - dependencies: - pend "~1.2.0" + ua-parser-js "^0.7.30" figgy-pudding@^3.5.1: version "3.5.2" @@ -8266,7 +8293,7 @@ fs-extra@^8.1.0: jsonfile "^4.0.0" universalify "^0.1.0" -fs-extra@^9.0.1, fs-extra@^9.1.0: +fs-extra@^9.0.1: version "9.1.0" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== @@ -8383,13 +8410,6 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= -getos@^3.2.1: - version "3.2.1" - resolved "https://registry.npmjs.org/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5" - integrity sha512-U56CfOK17OKgTVqozZjUKNdkfEv6jk5WISBJ8SHoagjE6L69zOwl3Z+O8myjY9MEW3i2HPWQBt/LTbCgcC973Q== - dependencies: - async "^3.2.0" - getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -8436,13 +8456,6 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" -global-dirs@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" - integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== - dependencies: - ini "2.0.0" - global-modules@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -8580,7 +8593,14 @@ graphql-tag@^2.10.1: dependencies: tslib "^2.1.0" -graphql-tag@^2.11.0, graphql-tag@^2.12.0: +graphql-tag@^2.11.0: + version "2.12.6" + resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.6.tgz#d441a569c1d2537ef10ca3d1633b48725329b5f1" + integrity sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg== + dependencies: + tslib "^2.1.0" + +graphql-tag@^2.12.0: version "2.12.4" resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.4.tgz#d34066688a4f09e72d6f4663c74211e9b4b7c4bf" integrity sha512-VV1U4O+9x99EkNpNmCUV5RZwq6MnK4+pGbRYWG+lA/m3uo7TSqJF81OkcOP148gFP6fzdl7JWYBrwWVTS9jXww== @@ -8893,7 +8913,7 @@ hmac-drbg@^1.0.1: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" -hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.1.0, hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2: +hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.1.0, hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw== @@ -9153,7 +9173,7 @@ immer@8.0.1: immutable@~3.7.6: version "3.7.6" resolved "https://registry.yarnpkg.com/immutable/-/immutable-3.7.6.tgz#13b4d3cb12befa15482a26fe1b2ebae640071e4b" - integrity sha1-E7TTyxK++hVIKib+Gy665kAHHks= + integrity sha512-AizQPcaofEtO11RZhPPHBOJRdo/20MKQF9mBLnVkBoyHi1/zXK8fzVdnEpSV9gxqtnh6Qomfp3F0xT5qP/vThw== import-cwd@^2.0.0: version "2.1.0" @@ -9185,6 +9205,11 @@ import-from@3.0.0: dependencies: resolve-from "^5.0.0" +import-from@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/import-from/-/import-from-4.0.0.tgz#2710b8d66817d232e16f4166e319248d3d5492e2" + integrity sha512-P9J71vT5nLlDeV8FHs5nNxaLbrpfAV5cF5srvbZfpwpcJoM/xZR3hiv+q+SAnuSmuGbXMWud063iIMx/V/EWZQ== + import-from@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/import-from/-/import-from-2.1.0.tgz#335db7f2a7affd53aaa471d4b8021dee36b7f3b1" @@ -9261,11 +9286,6 @@ inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= -ini@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - ini@^1.3.5, ini@~1.3.0: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" @@ -9317,6 +9337,13 @@ internmap@^1.0.0: resolved "https://registry.yarnpkg.com/internmap/-/internmap-1.0.1.tgz#0017cc8a3b99605f0302f2b198d272e015e5df95" integrity sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw== +invariant@^2.2.4: + version "2.2.4" + resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" + integrity sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA== + dependencies: + loose-envify "^1.0.0" + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" @@ -9420,7 +9447,7 @@ is-boolean-object@^1.1.0: dependencies: call-bind "^1.0.2" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@^1.1.5: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -9442,13 +9469,6 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-ci@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/is-ci/-/is-ci-3.0.0.tgz#c7e7be3c9d8eef7d0fa144390bd1e4b88dc4c994" - integrity sha512-kDXyttuLeslKAHYL/K28F2YkM3x5jvFPEw3yXbRptXydjD9rpLEz+C5K5iutY9ZiUu6AP41JdvRQwF4Iqs4ZCQ== - dependencies: - ci-info "^3.1.1" - is-color-stop@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-color-stop/-/is-color-stop-1.1.0.tgz#cfff471aee4dd5c9e158598fbe12967b5cdad345" @@ -9578,14 +9598,6 @@ is-hexadecimal@^1.0.0: resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz#cc35c97588da4bd49a8eedd6bc4082d44dcb23a7" integrity sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw== -is-installed-globally@~0.4.0: - version "0.4.0" - resolved "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" - integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== - dependencies: - global-dirs "^3.0.0" - is-path-inside "^3.0.2" - is-lower-case@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-lower-case/-/is-lower-case-2.0.2.tgz#1c0884d3012c841556243483aa5d522f47396d2a" @@ -9656,11 +9668,6 @@ is-path-inside@^2.1.0: dependencies: path-is-inside "^1.0.2" -is-path-inside@^3.0.2: - version "3.0.3" - resolved "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== - is-plain-obj@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" @@ -9723,11 +9730,6 @@ is-resolvable@^1.0.0: resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88" integrity sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg== -is-retry-allowed@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" - integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== - is-root@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-root/-/is-root-2.1.0.tgz#809e18129cf1129644302a4f8544035d51984a9c" @@ -10348,11 +10350,6 @@ joi@^17.3.0: "@sideway/formula" "^3.0.0" "@sideway/pinpoint" "^2.0.0" -join-component@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" - integrity sha1-uEF7dQZho5K+4sJTfGiyqdSXfNU= - js-cookie@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-2.2.1.tgz#69e106dc5d5806894562902aa5baec3744e9b2b8" @@ -10494,9 +10491,9 @@ json3@^3.3.3: integrity sha512-c7/8mbUsKigAbLkD5B010BK4D9LZm7A1pNItkEwiUZRpIN66exu/e7YQWysGun+TRKaJp8MhemM+VkfWv42aCA== json5@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" - integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== + version "1.0.2" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" + integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== dependencies: minimist "^1.2.0" @@ -10507,6 +10504,11 @@ json5@^2.1.2: dependencies: minimist "^1.2.5" +json5@^2.2.2: + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + jsonfile@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" @@ -10657,7 +10659,7 @@ latest-version@5.1.0: dependencies: package-json "^6.3.0" -lazy-ass@1.6.0, lazy-ass@^1.6.0: +lazy-ass@1.6.0: version "1.6.0" resolved "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= @@ -10965,7 +10967,7 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.once@^4.0.0, lodash.once@^4.1.1: +lodash.once@^4.0.0: version "4.1.1" resolved "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w= @@ -11189,15 +11191,6 @@ md5.js@^1.3.4: inherits "^2.0.1" safe-buffer "^5.1.2" -md5@^2.2.1: - version "2.3.0" - resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" - integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g== - dependencies: - charenc "0.0.2" - crypt "0.0.2" - is-buffer "~1.1.6" - mdast-util-definitions@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/mdast-util-definitions/-/mdast-util-definitions-4.0.0.tgz#c5c1a84db799173b4dcf7643cda999e440c24db2" @@ -11318,25 +11311,11 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== -memoize-one@^5.0.0: - version "5.2.1" - resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" - integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q== - memoize-one@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-6.0.0.tgz#b2591b871ed82948aee4727dc6abceeeac8c1045" integrity sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw== -memoizee@~0.2.5: - version "0.2.6" - resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.2.6.tgz#bb45a7ad02530082f1612671dab35219cd2e0741" - integrity sha1-u0WnrQJTAILxYSZx2rNSGc0uB0E= - dependencies: - es5-ext "~0.9.2" - event-emitter "~0.2.2" - next-tick "0.1.x" - memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" @@ -11560,15 +11539,10 @@ minimatch@3.0.4, minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.1.0.tgz#99df657a52574c21c9057497df742790b2b4c0de" - integrity sha1-md9lelJXTCHJBXSX33QnkLK0wN4= - minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + version "1.2.7" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" + integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== minipass-collect@^1.0.2: version "1.0.2" @@ -11662,7 +11636,7 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.4, mkdirp@^0.5.5, mkdirp@~0.5.1: +mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5, mkdirp@~0.5.1: version "0.5.5" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== @@ -11706,19 +11680,6 @@ monaco-editor@^0.28.1: resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.28.1.tgz#732788ff2172d59e6d436b206da8cac715413940" integrity sha512-P1vPqxB4B1ZFzTeR1ScggSp9/5NoQrLCq88fnlNUsuRAP1usEBN4TIpI2lw0AYIZNVIanHk0qwjze2uJwGOHUw== -monaco-yaml@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/monaco-yaml/-/monaco-yaml-3.2.1.tgz#45ce9f7f8140dc26249ac99eb0a9e5a9ab9beb87" - integrity sha512-2geAd5I7H1SMgwTHBuyPAPK9WTAzxbl9XwDl5h6NY6n9j4qnlLLQKK1i0P9cAmUiV2uaiViz69RLNWqVU5BVsg== - dependencies: - "@types/json-schema" "^7.0.0" - js-yaml "^4.0.0" - path-browserify "^1.0.0" - prettier "2.0.5" - vscode-languageserver-textdocument "^1.0.0" - vscode-languageserver-types "^3.0.0" - yaml-language-server-parser "^0.1.0" - move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -11741,7 +11702,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1: +ms@2.1.3, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -11818,11 +11779,6 @@ neo-async@^2.5.0, neo-async@^2.6.1, neo-async@^2.6.2: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== -next-tick@0.1.x: - version "0.1.0" - resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-0.1.0.tgz#1912cce8eb9b697d640fbba94f8f00dec3b94259" - integrity sha1-GRLM6OubaX1kD7upT48A3sO5Qlk= - next-tick@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" @@ -11857,6 +11813,13 @@ node-fetch@2.6.1, node-fetch@^2.6.1: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-fetch@2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -11918,6 +11881,11 @@ node-releases@^1.1.61, node-releases@^1.1.71: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.72.tgz#14802ab6b1039a79a0c7d662b610a5bbd76eacbe" integrity sha512-LLUo+PpH3dU6XizX3iVoubUNheF/owjXCZZ5yACDxNnPtgFuludV1ZL3ayK1kVep42Rmm0+R9/Y60NQbZ2bifw== +node-releases@^2.0.6: + version "2.0.8" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.8.tgz#0f349cdc8fcfa39a92ac0be9bc48b7706292b9ae" + integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A== + normalize-package-data@^2.3.2, normalize-package-data@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" @@ -12135,11 +12103,6 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" -onetime@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz#a1f7838f8314c516f05ecefcbc4ccfe04b4ed789" - integrity sha1-ofeDj4MUxRbwXs78vEzP4EtO14k= - onetime@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/onetime/-/onetime-2.0.1.tgz#067428230fd67443b2794b22bba528b6867962d4" @@ -12226,11 +12189,6 @@ os-tmpdir@~1.0.2: resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= -ospath@^1.2.2: - version "1.2.2" - resolved "https://registry.npmjs.org/ospath/-/ospath-1.2.2.tgz#1276639774a3f8ef2572f7fe4280e0ea4550c07b" - integrity sha1-EnZjl3Sj+O8lcvf+QoDg6kVQwHs= - p-cancelable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" @@ -12382,7 +12340,7 @@ parse-entities@^2.0.0: parse-filepath@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891" - integrity sha1-pjISf1Oq89FYdvWHLz/6x2PWyJE= + integrity sha512-FwdRXKCohSVeXqwtYonZTXtbGJKrn+HNyWDYVcp5yuJlesTwNH4rsmRZ+GrKAPJ5bLpRxESMeS+Rl0VCHRvB2Q== dependencies: is-absolute "^1.0.0" map-cache "^0.2.0" @@ -12434,11 +12392,6 @@ path-browserify@0.0.1: resolved "https://registry.yarnpkg.com/path-browserify/-/path-browserify-0.0.1.tgz#e6c4ddd7ed3aa27c68a20cc4e50e1a4ee83bbc4a" integrity sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ== -path-browserify@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-browserify/-/path-browserify-1.0.1.tgz#d98454a9c3753d5790860f16f68867b9e46be1fd" - integrity sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g== - path-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/path-case/-/path-case-3.0.4.tgz#9168645334eb942658375c56f80b4c0cb5f82c6f" @@ -12490,12 +12443,12 @@ path-parse@^1.0.6: path-root-regex@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/path-root-regex/-/path-root-regex-0.1.2.tgz#bfccdc8df5b12dc52c8b43ec38d18d72c04ba96d" - integrity sha1-v8zcjfWxLcUsi0PsONGNcsBLqW0= + integrity sha512-4GlJ6rZDhQZFE0DPVKh0e9jmZ5egZfxTkp7bcRDuPlJXbAwhxcl2dINPUAsjLdejqaLsCeg8axcLjIbvBjN4pQ== path-root@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/path-root/-/path-root-0.1.1.tgz#9a4a6814cac1c0cd73360a95f32083c8ea4745b7" - integrity sha1-mkpoFMrBwM1zNgqV8yCDyOpHRbc= + integrity sha512-QLcPegTHF11axjfojBIoDygmS2E3Lf+8+jI6wOVmNVenrKSo3mFdSGiIgdSHenczw3wPtlVMQaFVwGmM7BJdtg== dependencies: path-root-regex "^0.1.0" @@ -12541,22 +12494,22 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" -pend@~1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" - integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA= - performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +picocolors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" + integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== + picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== -pify@^2.0.0, pify@^2.2.0: +pify@^2.0.0: version "2.3.0" resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= @@ -13348,17 +13301,12 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.0.5.tgz#d6d56282455243f2f92cc1716692c08aa31522d4" - integrity sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg== - prettier@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.0.tgz#b6a5bf1284026ae640f17f7ff5658a7567fc0d18" integrity sha512-kXtO4s0Lz/DW/IJ9QdWhAf7/NmPWQXkFr/r/WkR3vyI+0v8amTDxiaQSLzs8NBlytfLWX/7uQUMIW677yLKl4w== -pretty-bytes@^5.3.0, pretty-bytes@^5.6.0: +pretty-bytes@^5.3.0: version "5.6.0" resolved "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== @@ -13618,11 +13566,6 @@ ramda@0.26.1: resolved "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz#8d41351eb8111c55353617fc3bbffad8e4d35d06" integrity sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ== -ramda@~0.27.1: - version "0.27.1" - resolved "https://registry.npmjs.org/ramda/-/ramda-0.27.1.tgz#66fc2df3ef873874ffc2da6aa8984658abacf5c9" - integrity sha512-PgIdVpn5y5Yns8vqb8FzBUEYn98V3xcPgawAkkgj0YJ0qDsnHCiNmZYfOGMgOvoB0eWFLpYbhxUR3mxfDIMvpw== - randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -14360,19 +14303,6 @@ react-scripts@4.0.3: optionalDependencies: fsevents "^2.1.3" -react-select@^5.1.0: - version "5.2.1" - resolved "https://registry.yarnpkg.com/react-select/-/react-select-5.2.1.tgz#416c25c6b79b94687702374e019c4f2ed9d159d6" - integrity sha512-OOyNzfKrhOcw/BlembyGWgdlJ2ObZRaqmQppPFut1RptJO423j+Y+JIsmxkvsZ4D/3CpOmwIlCvWbbAWEdh12A== - dependencies: - "@babel/runtime" "^7.12.0" - "@emotion/cache" "^11.4.0" - "@emotion/react" "^11.1.1" - "@types/react-transition-group" "^4.4.0" - memoize-one "^5.0.0" - prop-types "^15.6.0" - react-transition-group "^4.3.0" - react-side-effect@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/react-side-effect/-/react-side-effect-2.1.2.tgz#dc6345b9e8f9906dc2eeb68700b615e0b4fe752a" @@ -14389,25 +14319,6 @@ react-syntax-highlighter@^15.4.4: prismjs "^1.22.0" refractor "^3.2.0" -react-timezone-select@^1.1.15: - version "1.1.15" - resolved "https://registry.yarnpkg.com/react-timezone-select/-/react-timezone-select-1.1.15.tgz#928b51028dfb193cefc10215426aa8f493fe3137" - integrity sha512-a+GD66LAkpS8zl3F4wfEwocJU8I3DmDKhZSzhNjXwcSal+jIKLw8RObesiOptfuktgGMDi12esv1FY2k5zZoBw== - dependencies: - react-select "^5.1.0" - spacetime "^6.16.3" - timezone-soft "^1.3.1" - -react-transition-group@^4.3.0: - version "4.4.2" - resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.2.tgz#8b59a56f09ced7b55cbd53c36768b922890d5470" - integrity sha512-/RNYfRAMlZwDSr6z4zNKV6xu53/e2BuaBbGhbyYIXTrmgu/bGHzmqOs7mJSJBHy9Ud+ApHx3QjrkKSp1pxvlFg== - dependencies: - "@babel/runtime" "^7.5.5" - dom-helpers "^5.0.1" - loose-envify "^1.4.0" - prop-types "^15.6.2" - react-visibility-sensor@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/react-visibility-sensor/-/react-visibility-sensor-5.1.1.tgz#5238380960d3a0b2be0b7faddff38541e337f5a9" @@ -14693,35 +14604,14 @@ relateurl@^0.2.7: resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk= -relay-compiler@10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/relay-compiler/-/relay-compiler-10.1.0.tgz#fb4672cdbe9b54869a3a79759edd8c2d91609cbe" - integrity sha512-HPqc3N3tNgEgUH5+lTr5lnLbgnsZMt+MRiyS0uAVNhuPY2It0X1ZJG+9qdA3L9IqKFUNwVn6zTO7RArjMZbARQ== - dependencies: - "@babel/core" "^7.0.0" - "@babel/generator" "^7.5.0" - "@babel/parser" "^7.0.0" - "@babel/runtime" "^7.0.0" - "@babel/traverse" "^7.0.0" - "@babel/types" "^7.0.0" - babel-preset-fbjs "^3.3.0" - chalk "^4.0.0" - fb-watchman "^2.0.0" - fbjs "^3.0.0" - glob "^7.1.1" - immutable "~3.7.6" - nullthrows "^1.1.1" - relay-runtime "10.1.0" - signedsource "^1.0.0" - yargs "^15.3.1" - -relay-runtime@10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/relay-runtime/-/relay-runtime-10.1.0.tgz#4753bf36e95e8d862cef33608e3d98b4ed730d16" - integrity sha512-bxznLnQ1ST6APN/cFi7l0FpjbZVchWQjjhj9mAuJBuUqNNCh9uV+UTRhpQF7Q8ycsPp19LHTpVyGhYb0ustuRQ== +relay-runtime@12.0.0: + version "12.0.0" + resolved "https://registry.yarnpkg.com/relay-runtime/-/relay-runtime-12.0.0.tgz#1e039282bdb5e0c1b9a7dc7f6b9a09d4f4ff8237" + integrity sha512-QU6JKr1tMsry22DXNy9Whsq5rmvwr3LSZiiWV/9+DFpuTWvp+WFhobWMc8TC4OjKFfNhEZy7mOiqUAn5atQtug== dependencies: "@babel/runtime" "^7.0.0" fbjs "^3.0.0" + invariant "^2.2.4" remark-gfm@1.0.0: version "1.0.0" @@ -14755,11 +14645,6 @@ remove-trailing-separator@^1.0.1: resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" integrity sha1-wkvOKig62tW8P1jg1IJJuSN52O8= -remove-trailing-slash@^0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz#be2285a59f39c74d1bce4f825950061915e3780d" - integrity sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA== - remove-trailing-spaces@^1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/remove-trailing-spaces/-/remove-trailing-spaces-1.0.8.tgz#4354d22f3236374702f58ee373168f6d6887ada7" @@ -14791,13 +14676,6 @@ replaceall@^0.1.6: resolved "https://registry.yarnpkg.com/replaceall/-/replaceall-0.1.6.tgz#81d81ac7aeb72d7f5c4942adf2697a3220688d8e" integrity sha1-gdgax663LX9cSUKt8ml6MiBojY4= -request-progress@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/request-progress/-/request-progress-3.0.0.tgz#4ca754081c7fec63f505e4faa825aa06cd669dbe" - integrity sha1-TKdUCBx/7GP1BeT6qCWqBs1mnb4= - dependencies: - throttleit "^1.0.0" - request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" @@ -14824,11 +14702,6 @@ request@^2.88.2: tunnel-agent "^0.6.0" uuid "^3.3.2" -require-dir@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/require-dir/-/require-dir-0.1.0.tgz#81e01e299faf5b74c34b6594f8e5add5985ddec5" - integrity sha1-geAeKZ+vW3TDS2WU+OWt1Zhd3sU= - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -14859,7 +14732,7 @@ resize-observer-polyfill@1.5.0: resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.0.tgz#660ff1d9712a2382baa2cad450a4716209f9ca69" integrity sha512-M2AelyJDVR/oLnToJLtuDJRBBWUGUvvGigj1411hXhAdyFWqMaqHp7TixW3FpiLuVaikIcR1QL+zqoJoZlOgpg== -resize-observer-polyfill@1.5.1, resize-observer-polyfill@^1.5.0, resize-observer-polyfill@^1.5.1: +resize-observer-polyfill@^1.5.0, resize-observer-polyfill@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464" integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg== @@ -14950,14 +14823,6 @@ responselike@^1.0.2: dependencies: lowercase-keys "^1.0.0" -restore-cursor@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz#34661f46886327fed2991479152252df92daa541" - integrity sha1-NGYfRohjJ/7SmRR5FSJS35LapUE= - dependencies: - exit-hook "^1.0.0" - onetime "^1.0.0" - restore-cursor@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-2.0.0.tgz#9f7ee287f82fd326d4fd162923d62129eee0dfaf" @@ -15437,7 +15302,7 @@ signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: signedsource@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/signedsource/-/signedsource-1.0.0.tgz#1ddace4981798f93bd833973803d80d52e93ad6a" - integrity sha1-HdrOSYF5j5O9gzlzgD2A1S6TrWo= + integrity sha512-6+eerH9fEnNmi/hyM1DXcRK3pWdoMQtlkQ+ns0ntzunjKqp5i3sKCc80ym8Fib3iaYhdJUOPdhlJWj1tvge2Ww== simple-swizzle@^0.2.2: version "0.2.2" @@ -15633,11 +15498,6 @@ space-separated-tokens@^1.0.0, space-separated-tokens@^1.1.0: resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz#85f32c3d10d9682007e917414ddc5c26d1aa6899" integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA== -spacetime@^6.16.3: - version "6.16.3" - resolved "https://registry.yarnpkg.com/spacetime/-/spacetime-6.16.3.tgz#86d3b05db33421a9ee478b1f2ca025582fc61fcf" - integrity sha512-JQEfj3VHT1gU1IMV5NvhgAP8P+2mDFd84ZCiHN//dp6hRKmuW0IizHissy62lO0nilfFjVhnoSaMC7te+Y5f4A== - spdx-correct@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" @@ -16082,11 +15942,6 @@ stylehacks@^4.0.0: postcss "^7.0.0" postcss-selector-parser "^3.0.0" -stylis@^4.0.10: - version "4.0.10" - resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.0.10.tgz#446512d1097197ab3f02fb3c258358c3f7a14240" - integrity sha512-m3k+dk7QeJw660eIKRRn3xPF6uuvHs/FFzjX3HQ5ove0qYsiygoAhwn5a3IYKaZPo5LrYD0rfVmtv1gNY1uYwg== - subscriptions-transport-ws@^0.9.18: version "0.9.18" resolved "https://registry.yarnpkg.com/subscriptions-transport-ws/-/subscriptions-transport-ws-0.9.18.tgz#bcf02320c911fbadb054f7f928e51c6041a37b97" @@ -16124,13 +15979,6 @@ supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0: dependencies: has-flag "^4.0.0" -supports-color@^8.1.1: - version "8.1.1" - resolved "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - supports-hyperlinks@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.2.0.tgz#4f77b42488765891774b70c79babd87f9bd594bb" @@ -16319,11 +16167,6 @@ throat@^5.0.0: resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA== -throttleit@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" - integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw= - through2@^2.0.0: version "2.0.5" resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd" @@ -16332,7 +16175,7 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through@2, through@^2.3.6, through@~2.3, through@~2.3.1, through@~2.3.4: +through@2, through@^2.3.6, through@~2.3, through@~2.3.1: version "2.3.8" resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -16349,11 +16192,6 @@ timers-browserify@^2.0.4: dependencies: setimmediate "^1.0.4" -timezone-soft@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/timezone-soft/-/timezone-soft-1.3.1.tgz#0e994067cbccc76a9c16b71fd8f2f94394be5b0d" - integrity sha512-mphMogFJzQy6UIpl/UgKLSNbhmLtdgbz866TnqJ/CnWnc+7dsNyAe8nPwVdOW3Mf8nT0lA32MW/gYhAl4/RkVg== - timsort@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/timsort/-/timsort-0.3.0.tgz#405411a8e7e6339fe64db9a234de11dc31e02bd4" @@ -16393,13 +16231,6 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" -tmp@~0.2.1: - version "0.2.1" - resolved "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" - integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== - dependencies: - rimraf "^3.0.0" - tmpl@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" @@ -16486,6 +16317,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + trough@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" @@ -16496,13 +16332,6 @@ tryer@^1.0.1: resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8" integrity sha512-c3zayb8/kWWpycWYg87P71E1S1ZL6b6IJxfb5fvsUgsf0S2MVGaDhDXXjDMpdCpfWXqptc+4mXwmiy1ypXqRAA== -ts-invariant@^0.4.0: - version "0.4.4" - resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.4.4.tgz#97a523518688f93aafad01b0e80eb803eb2abd86" - integrity sha512-uEtWkFM/sdZvRNNDL3Ehu4WVpwaulhwQszV8mrtcdeE8nN00BV9mAmQ88RkrBhFgl9gMgvjJLAQcZbnPXI9mlA== - dependencies: - tslib "^1.9.3" - ts-invariant@^0.7.0: version "0.7.3" resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.7.3.tgz#13aae22a4a165393aaf5cecdee45ef4128d358b8" @@ -16542,7 +16371,7 @@ tsconfig-paths@^3.9.0: minimist "^1.2.0" strip-bom "^3.0.0" -tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: +tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== @@ -16552,7 +16381,7 @@ tslib@^2, tslib@^2.0.3, tslib@^2.1.0, tslib@~2.2.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.2.0.tgz#fb2c475977e35e241311ede2693cee1ec6698f5c" integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w== -tslib@^2.0.0: +tslib@^2.0.0, tslib@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e" integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA== @@ -16567,6 +16396,11 @@ tslib@~2.1.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== +tslib@~2.3.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" + integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== + tsutils@^3.17.1: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" @@ -16670,10 +16504,10 @@ typescript@^4.1.3: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.2.4.tgz#8610b59747de028fda898a8aef0e103f156d0961" integrity sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg== -ua-parser-js@^0.7.18: - version "0.7.28" - resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.28.tgz#8ba04e653f35ce210239c64661685bf9121dec31" - integrity sha512-6Gurc1n//gjp9eQNXjD9O3M/sMwVtN5S8Lv9bvOYBfKfDNiIIhqiyi01vMBO45u4zkDE420w/e0se7Vs+sIg+g== +ua-parser-js@^0.7.30: + version "0.7.32" + resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.32.tgz#cd8c639cdca949e30fa68c44b7813ef13e36d211" + integrity sha512-f9BESNVhzlhEFf2CHMSj40NWOjYPl1YKYbrvIr/hFTDEmLq7SRbWvm7FcdcpCYT95zrOhC7gZSxjdnnTpBcwVw== unbox-primitive@^1.0.1: version "1.0.1" @@ -16688,7 +16522,7 @@ unbox-primitive@^1.0.1: unc-path-regex@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" - integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo= + integrity sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg== unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" @@ -16854,16 +16688,19 @@ unset-value@^1.0.0: has-value "^0.3.1" isobject "^3.0.0" -untildify@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" - integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== - upath@^1.1.1, upath@^1.1.2, upath@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== +update-browserslist-db@^1.0.9: + version "1.0.10" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3" + integrity sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ== + dependencies: + escalade "^3.1.1" + picocolors "^1.0.0" + upper-case-first@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/upper-case-first/-/upper-case-first-2.0.2.tgz#992c3273f882abd19d1e02894cc147117f844324" @@ -16997,7 +16834,7 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== -uuid@^3.0.0, uuid@^3.2.1, uuid@^3.3.2, uuid@^3.4.0: +uuid@^3.0.0, uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== @@ -17096,27 +16933,6 @@ vm-browserify@^1.0.1: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ== -vscode-languageserver-textdocument@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.3.tgz#879f2649bfa5a6e07bc8b392c23ede2dfbf43eff" - integrity sha512-ynEGytvgTb6HVSUwPJIAZgiHQmPCx8bZ8w5um5Lz+q5DjP0Zj8wTFhQpyg8xaMvefDytw2+HH5yzqS+FhsR28A== - -vscode-languageserver-types@^3.0.0: - version "3.16.0" - resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz#ecf393fc121ec6974b2da3efb3155644c514e247" - integrity sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA== - -vx@^0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/vx/-/vx-0.0.1.tgz#db2c2686f41afd6332f94762004b1a34d7fe5cad" - integrity sha1-2ywmhvQa/WMy+UdiAEsaNNf+XK0= - dependencies: - bluebird "~2.2.2" - cli-color "~0.2.3" - minimist "^0.1.0" - require-dir "^0.1.0" - through "~2.3.4" - w3c-hr-time@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" @@ -17191,6 +17007,11 @@ web-vitals@^0.2.4: resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-0.2.4.tgz#ec3df43c834a207fd7cdefd732b2987896e08511" integrity sha512-6BjspCO9VriYy12z356nL6JBS0GYeEcA457YyRzD+dD6XYCQ75NKhcOHUMHentOE7OcVCIXXDvOm0jKFfQG2Gg== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -17344,6 +17165,14 @@ whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + whatwg-url@^8.0.0, whatwg-url@^8.5.0: version "8.5.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.5.0.tgz#7752b8464fc0903fec89aa9846fc9efe07351fd3" @@ -17669,11 +17498,6 @@ yaml-ast-parser@^0.0.43: resolved "https://registry.yarnpkg.com/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb" integrity sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A== -yaml-language-server-parser@^0.1.0: - version "0.1.3" - resolved "https://registry.yarnpkg.com/yaml-language-server-parser/-/yaml-language-server-parser-0.1.3.tgz#f0e9082068291c7c330eefa1f3c9f1b4c3c54183" - integrity sha512-xD2I+6M/vqQvcy4ded8JpXUaDHXmZMdhIO3OpuiFxstutwnW4whrfDzNcrsfXVdgMWqOUpdv3747Q081PFN1+g== - yaml@^1.10.0, yaml@^1.7.2: version "1.10.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" @@ -17754,14 +17578,6 @@ yargs@^16.1.1, yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yauzl@^2.10.0: - version "2.10.0" - resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk= - dependencies: - buffer-crc32 "~0.2.3" - fd-slicer "~1.1.0" - yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" @@ -17772,15 +17588,7 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -zen-observable-ts@^0.8.21: - version "0.8.21" - resolved "https://registry.yarnpkg.com/zen-observable-ts/-/zen-observable-ts-0.8.21.tgz#85d0031fbbde1eba3cd07d3ba90da241215f421d" - integrity sha512-Yj3yXweRc8LdRMrCC8nIc4kkjWecPAUVh0TI0OUrWXx6aX790vLcDlWca6I4vsyCGH3LpWxq0dJRcMOFoVqmeg== - dependencies: - tslib "^1.9.3" - zen-observable "^0.8.0" - -zen-observable@^0.8.0, zen-observable@^0.8.14: +zen-observable@^0.8.14: version "0.8.15" resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15" integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ== diff --git a/docker/kafka-setup/kafka-topic-workers.sh b/docker/kafka-setup/kafka-topic-workers.sh index b132a04201c60..14b2d73effda1 100644 --- a/docker/kafka-setup/kafka-topic-workers.sh +++ b/docker/kafka-setup/kafka-topic-workers.sh @@ -63,3 +63,4 @@ for ((i=1;i<=$WORKERS;i++)); do echo will start $i work $i & done + diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index 7c8f634a70d83..586a85f4d653a 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1538,7 +1538,6 @@ "@docusaurus/types" "2.2.0" "@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": - name "@docusaurus/react-loadable" version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -5357,9 +5356,9 @@ json-schema-traverse@^1.0.0: integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== json5@^2.1.2, json5@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c" - integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== jsonfile@^6.0.1: version "6.1.0" diff --git a/docs/cli.md b/docs/cli.md index 6df444fca9358..4a8be2a930ee6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -220,6 +220,9 @@ Telemetry is enabled by default, and the `telemetry` command lets you toggle the ### delete The `delete` command allows you to delete metadata from DataHub. Read this [guide](./how/delete-metadata.md) to understand how you can delete metadata from DataHub. +:::info +Deleting metadata using DataHub's CLI and GraphQL API is a simple, systems-level action. If you attempt to delete an Entity with children, such as a Container, it will not automatically delete the children, you will instead need to delete each child by URN in addition to deleting the parent. +::: ```console datahub delete --urn "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" --soft diff --git a/docs/how/delete-metadata.md b/docs/how/delete-metadata.md index 7c5009ecf9baf..5a1c61aa6b615 100644 --- a/docs/how/delete-metadata.md +++ b/docs/how/delete-metadata.md @@ -10,6 +10,10 @@ Read on to find out how to perform these kinds of deletes. _Note: Deleting metadata should only be done with care. Always use `--dry-run` to understand what will be deleted before proceeding. Prefer soft-deletes (`--soft`) unless you really want to nuke metadata rows. Hard deletes will actually delete rows in the primary store and recovering them will require using backups of the primary metadata store. Make sure you understand the implications of issuing soft-deletes versus hard-deletes before proceeding._ + +:::info +Deleting metadata using DataHub's CLI and GraphQL API is a simple, systems-level action. If you attempt to delete an Entity with children, such as a Domain, it will not delete those children, you will instead need to delete each child by URN in addition to deleting the parent. +::: ## Delete By Urn To delete all the data related to a single entity, run diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 665013cf46efc..a877c8ee22d78 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -7,6 +7,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes - #6742 The metadata file sink's output format no longer contains nested JSON strings for MCP aspects, but instead unpacks the stringified JSON into a real JSON object. The previous sink behavior can be recovered using the `legacy_nested_json_string` option. The file source is backwards compatible and supports both formats. +- #6901 The `env` and `database_alias` fields have been marked deprecated across all sources. We recommend using `platform_instance` where possible instead. ### Potential Downtime diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 3f45039025877..dfd4f85cc4aa2 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -72,6 +72,7 @@ public class Constants { public static final String SIBLINGS_ASPECT_NAME = "siblings"; public static final String ORIGIN_ASPECT_NAME = "origin"; public static final String INPUT_FIELDS_ASPECT_NAME = "inputFields"; + public static final String EMBED_ASPECT_NAME = "embed"; // User public static final String CORP_USER_KEY_ASPECT_NAME = "corpUserKey"; diff --git a/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md b/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md index da1be4be8a6a3..e9e12e36613dd 100644 --- a/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md +++ b/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md @@ -38,7 +38,7 @@ The business glossary source file should be a `.yml` file with the following top - **custom_properties**: A map of key/value pairs of arbitrary custom properties - **domain**: (optional) domain name or domain urn -You can also view an example business glossary file checked in [here](../../../examples/bootstrap_data/business_glossary.yml) +You can also view an example business glossary file checked in [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/bootstrap_data/business_glossary.yml) ## Compatibility diff --git a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md index ad33b147ffcb9..e6e714055fb9d 100644 --- a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md +++ b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md @@ -16,7 +16,7 @@ grant usage on DATABASE "" to role datahub_role; grant usage on all schemas in database "" to role datahub_role; grant usage on future schemas in database "" to role datahub_role; -// If you are NOT using Snowflake Profiling or Classification feature: Grant references privileges to your tables and views +// If you are NOT using Snowflake Profiling or Classification feature: Grant references privileges to your tables and views grant references on all tables in database "" to role datahub_role; grant references on future tables in database "" to role datahub_role; grant references on all external tables in database "" to role datahub_role; @@ -30,10 +30,10 @@ grant select on future tables in database "" to role datahub_role grant select on all external tables in database "" to role datahub_role; grant select on future external tables in database "" to role datahub_role; -// Create a new DataHub user and assign the DataHub role to it +// Create a new DataHub user and assign the DataHub role to it create user datahub_user display_name = 'DataHub' password='' default_role = datahub_role default_warehouse = ''; -// Grant the datahub_role to the new DataHub user. +// Grant the datahub_role to the new DataHub user. grant role datahub_role to user datahub_user; ``` @@ -50,7 +50,7 @@ grant usage on schema ""."" to role datahub_role; This represents the bare minimum privileges required to extract databases, schemas, views, tables from Snowflake. -If you plan to enable extraction of table lineage, via the `include_table_lineage` config flag or extraction of usage statistics, via the `include_usage_stats` config, you'll also need to grant access to the [Account Usage](https://docs.snowflake.com/en/sql-reference/account-usage.html) system tables, using which the DataHub source extracts information. This can be done by granting access to the `snowflake` database. +If you plan to enable extraction of table lineage, via the `include_table_lineage` config flag, extraction of usage statistics, via the `include_usage_stats` config, or extraction of tags (without lineage), via the `extract_tags` config, you'll also need to grant access to the [Account Usage](https://docs.snowflake.com/en/sql-reference/account-usage.html) system tables, using which the DataHub source extracts information. This can be done by granting access to the `snowflake` database. ```sql grant imported privileges on database snowflake to role datahub_role; diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index b8eb1da0cc1dc..036739e60e47d 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -24,7 +24,8 @@ def get_long_description(): "mypy_extensions>=0.4.3", # Actual dependencies. "typing-inspect", - "pydantic>=1.5.1,<1.10.3", + # pydantic 1.10.3 is incompatible with typing-extensions 4.1.1 - https://github.com/pydantic/pydantic/issues/4885 + "pydantic>=1.5.1,!=1.10.3", "mixpanel>=4.9.0", } @@ -217,7 +218,7 @@ def get_long_description(): data_lake_profiling = { "pydeequ>=1.0.1", - "pyspark>=3.0.3", + "pyspark==3.0.3", } delta_lake = { @@ -341,9 +342,9 @@ def get_long_description(): "trino": sql_common | trino, "starburst-trino-usage": sql_common | usage_common | trino, "nifi": {"requests", "packaging"}, - "powerbi": microsoft_common | {"lark[regex]==1.1.4"}, + "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"}, "powerbi-report-server": powerbi_report_server, - "vertica": sql_common | {"sqlalchemy-vertica[vertica-python]==0.0.5"}, + "vertica": sql_common | {"sqlalchemy-vertica-dialect[vertica-python]==0.1.4"}, "unity-catalog": databricks_cli | {"requests"}, } @@ -393,7 +394,7 @@ def get_long_description(): "mypy==0.991", # pydantic 1.8.2 is incompatible with mypy 0.910. # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910. - "pydantic >=1.9.0", + "pydantic>=1.9.0", "pytest>=6.2.2", "pytest-asyncio>=0.16.0", "pytest-cov>=2.8.1", diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index cb479d2a86009..6cf793bd29e61 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -411,9 +411,9 @@ def get_urns_by_filter( entities_yielded += 1 log.debug(f"yielding {x['entity']}") yield x["entity"] - assert ( - entities_yielded == num_entities - ), "Did not delete all entities, try running this command again!" + log.warning( + f"Discrepancy in entities yielded {entities_yielded} and num entities {num_entities}. This means all entities may not have been deleted." + ) else: log.error(f"Failed to execute search query with {str(response.content)}") response.raise_for_status() diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py index 8e9b903d9f1ad..51037c684ed85 100644 --- a/metadata-ingestion/src/datahub/cli/delete_cli.py +++ b/metadata-ingestion/src/datahub/cli/delete_cli.py @@ -84,7 +84,9 @@ def delete_for_registry( @click.option("--urn", required=False, type=str, help="the urn of the entity") @click.option( "-a", + # option with `_` is inconsistent with rest of CLI but kept for backward compatibility "--aspect_name", + "--aspect-name", required=False, type=str, help="the aspect name associated with the entity(only for timeseries aspects)", @@ -106,11 +108,13 @@ def delete_for_registry( "-p", "--platform", required=False, type=str, help="the platform of the entity" ) @click.option( + # option with `_` is inconsistent with rest of CLI but kept for backward compatibility "--entity_type", + "--entity-type", required=False, type=str, default="dataset", - help="the entity_type of the entity", + help="the entity type of the entity", ) @click.option("--query", required=False, type=str) @click.option( diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 78b3eb71e7ec2..1938c60880da7 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -158,6 +158,10 @@ class IgnorableError(MetaError): """An error that can be ignored.""" +class ConfigurationWarning(Warning): + """A configuration warning.""" + + class ConfigurationMechanism(ABC): @abstractmethod def load_config(self, config_fp: IO) -> dict: diff --git a/metadata-ingestion/src/datahub/configuration/github.py b/metadata-ingestion/src/datahub/configuration/github.py index 3efaedcb7d774..b99fd3f5c797d 100644 --- a/metadata-ingestion/src/datahub/configuration/github.py +++ b/metadata-ingestion/src/datahub/configuration/github.py @@ -12,7 +12,7 @@ class GitHubReference(ConfigModel): ) branch: str = Field( "main", - description="Branch on which your files live by default. Typically main or master.", + description="Branch on which your files live by default. Typically main or master. This can also be a commit hash.", ) base_url: str = Field( "https://github.com", @@ -73,3 +73,15 @@ def auto_infer_from_repo(cls, v: Optional[str], values: Dict[str, Any]) -> str: if v is None: return f"git@github.com:{values.get('repo')}" return v + + @property + def branch_for_clone(self) -> Optional[str]: + # If branch was manually set, we should use it. Otherwise return None. + # We do this because we want to use the default branch unless they override it. + # While our default for branch is "main", they could be using "master" or something else. + # It's ok if the URLs we generate are slightly incorrect, but changing branch to be + # required would be a breaking change. + + if "branch" in self.__fields_set__: + return self.branch + return None diff --git a/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py b/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py new file mode 100644 index 0000000000000..10d0bc67e2136 --- /dev/null +++ b/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py @@ -0,0 +1,20 @@ +import warnings +from typing import Optional, Type + +import pydantic + +from datahub.configuration.common import ConfigurationWarning + + +def pydantic_field_deprecated(field: str, message: Optional[str] = None) -> classmethod: + if message: + output = message + else: + output = f"{field} is deprecated and will be removed in a future release. Please remove it from your config." + + def _validate_deprecated(cls: Type, values: dict) -> dict: + if field in values: + warnings.warn(output, ConfigurationWarning, stacklevel=2) + return values + + return pydantic.root_validator(pre=True, allow_reuse=True)(_validate_deprecated) diff --git a/metadata-ingestion/src/datahub/configuration/source_common.py b/metadata-ingestion/src/datahub/configuration/source_common.py index 7c10fc4613f36..09d86f93de98d 100644 --- a/metadata-ingestion/src/datahub/configuration/source_common.py +++ b/metadata-ingestion/src/datahub/configuration/source_common.py @@ -4,6 +4,7 @@ from pydantic.fields import Field from datahub.configuration.common import ConfigModel, ConfigurationError +from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated from datahub.metadata.schema_classes import FabricTypeClass DEFAULT_ENV = FabricTypeClass.PROD @@ -39,6 +40,11 @@ class EnvBasedSourceConfigBase(ConfigModel): description="The environment that all assets produced by this connector belong to", ) + _env_deprecation = pydantic_field_deprecated( + "env", + "env is deprecated and will be removed in a future release. Please use platform_instance instead.", + ) + @validator("env") def env_must_be_one_of(cls, v: str) -> str: if v.upper() not in ALL_ENV_TYPES: diff --git a/metadata-ingestion/src/datahub/configuration/validate_field_removal.py b/metadata-ingestion/src/datahub/configuration/validate_field_removal.py index f3b45e71ddb85..8f60953760808 100644 --- a/metadata-ingestion/src/datahub/configuration/validate_field_removal.py +++ b/metadata-ingestion/src/datahub/configuration/validate_field_removal.py @@ -3,6 +3,8 @@ import pydantic +from datahub.configuration.common import ConfigurationWarning + def pydantic_removed_field( field: str, @@ -13,7 +15,7 @@ def _validate_field_rename(cls: Type, values: dict) -> dict: if print_warning: warnings.warn( f"The {field} was removed, please remove it from your recipe.", - UserWarning, + ConfigurationWarning, stacklevel=2, ) values.pop(field) diff --git a/metadata-ingestion/src/datahub/configuration/validate_field_rename.py b/metadata-ingestion/src/datahub/configuration/validate_field_rename.py index 496bac6517797..4dc5a227ac834 100644 --- a/metadata-ingestion/src/datahub/configuration/validate_field_rename.py +++ b/metadata-ingestion/src/datahub/configuration/validate_field_rename.py @@ -3,6 +3,8 @@ import pydantic +from datahub.configuration.common import ConfigurationWarning + _T = TypeVar("_T") @@ -25,8 +27,8 @@ def _validate_field_rename(cls: Type, values: dict) -> dict: else: if print_warning: warnings.warn( - f"The {old_name} is deprecated, please use {new_name} instead.", - UserWarning, + f"{old_name} is deprecated, please use {new_name} instead.", + ConfigurationWarning, stacklevel=2, ) values[new_name] = transform(values.pop(old_name)) diff --git a/metadata-ingestion/src/datahub/emitter/mcp.py b/metadata-ingestion/src/datahub/emitter/mcp.py index a1cdf93198b04..1356f58f05103 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp.py +++ b/metadata-ingestion/src/datahub/emitter/mcp.py @@ -1,6 +1,6 @@ import dataclasses import json -from typing import TYPE_CHECKING, Optional, Tuple, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Union from datahub.emitter.aspect import ASPECT_MAP, TIMESERIES_ASPECT_MAP from datahub.emitter.serialization_helper import post_json_transform, pre_json_transform @@ -99,6 +99,12 @@ def __post_init__(self) -> None: f"aspectName {self.aspectName} does not match aspect type {type(self.aspect)} with name {self.aspect.get_aspect_name()}" ) + @classmethod + def construct_many( + cls, entityUrn: str, aspects: List[Optional[_Aspect]] + ) -> List["MetadataChangeProposalWrapper"]: + return [cls(entityUrn=entityUrn, aspect=aspect) for aspect in aspects if aspect] + def make_mcp(self) -> MetadataChangeProposalClass: serializedEntityKeyAspect: Union[None, GenericAspectClass] = None if isinstance(self.entityKeyAspect, DictWrapper): diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index f5d63c68d137e..7d0d6f24dbdb6 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -2,6 +2,7 @@ import json from typing import Any, Dict, Iterable, List, Optional, TypeVar +from deprecated import deprecated from pydantic.fields import Field from pydantic.main import BaseModel @@ -18,7 +19,6 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.container import ContainerProperties from datahub.metadata.schema_classes import ( - ChangeTypeClass, ContainerClass, DomainsClass, GlobalTagsClass, @@ -31,7 +31,6 @@ TagAssociationClass, _Aspect, ) -from datahub.utilities.urns.urn import guess_entity_type def _stable_guid_from_dict(d: dict) -> str: @@ -129,24 +128,18 @@ def default(self, obj: Any) -> Any: def add_domain_to_entity_wu( - entity_type: str, entity_urn: str, domain_urn: str + entity_urn: str, domain_urn: str ) -> Iterable[MetadataWorkUnit]: - mcp = MetadataChangeProposalWrapper( - entityType=entity_type, - changeType=ChangeTypeClass.UPSERT, + yield MetadataChangeProposalWrapper( entityUrn=f"{entity_urn}", aspect=DomainsClass(domains=[domain_urn]), - ) - wu = MetadataWorkUnit(id=f"{domain_urn}-to-{entity_urn}", mcp=mcp) - yield wu + ).as_workunit() def add_owner_to_entity_wu( entity_type: str, entity_urn: str, owner_urn: str ) -> Iterable[MetadataWorkUnit]: - mcp = MetadataChangeProposalWrapper( - entityType=entity_type, - changeType=ChangeTypeClass.UPSERT, + yield MetadataChangeProposalWrapper( entityUrn=f"{entity_urn}", aspect=OwnershipClass( owners=[ @@ -156,26 +149,22 @@ def add_owner_to_entity_wu( ) ] ), - ) - wu = MetadataWorkUnit(id=f"{owner_urn}-to-{entity_urn}", mcp=mcp) - yield wu + ).as_workunit() def add_tags_to_entity_wu( entity_type: str, entity_urn: str, tags: List[str] ) -> Iterable[MetadataWorkUnit]: - mcp = MetadataChangeProposalWrapper( + yield MetadataChangeProposalWrapper( entityType=entity_type, - changeType=ChangeTypeClass.UPSERT, entityUrn=f"{entity_urn}", aspect=GlobalTagsClass( tags=[TagAssociationClass(f"urn:li:tag:{tag}") for tag in tags] ), - ) - wu = MetadataWorkUnit(id=f"tags-to-{entity_urn}", mcp=mcp) - yield wu + ).as_workunit() +@deprecated("use MetadataChangeProposalWrapper(...).as_workunit() instead") def wrap_aspect_as_workunit( entityName: str, entityUrn: str, @@ -210,9 +199,7 @@ def gen_containers( container_urn = make_container_urn( guid=container_key.guid(), ) - mcp = MetadataChangeProposalWrapper( - entityType="container", - changeType=ChangeTypeClass.UPSERT, + yield MetadataChangeProposalWrapper( entityUrn=f"{container_urn}", # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), aspect=ContainerProperties( @@ -229,51 +216,32 @@ def gen_containers( if last_modified is not None else None, ), - ) - wu = MetadataWorkUnit(id=f"container-info-{name}-{container_urn}", mcp=mcp) - yield wu + ).as_workunit() # add status - yield wrap_aspect_as_workunit( - entityName="container", + yield MetadataChangeProposalWrapper( entityUrn=f"{container_urn}", aspect=StatusClass(removed=False), - aspectName=StatusClass.get_aspect_name(), - ) + ).as_workunit() - mcp = MetadataChangeProposalWrapper( - entityType="container", - changeType=ChangeTypeClass.UPSERT, + yield MetadataChangeProposalWrapper( entityUrn=f"{container_urn}", - # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), aspect=DataPlatformInstance( platform=f"{make_data_platform_urn(container_key.platform)}", instance=f"{make_dataplatform_instance_urn(container_key.platform, container_key.instance)}" if container_key.instance else None, ), - ) - wu = MetadataWorkUnit( - id=f"container-platforminstance-{name}-{container_urn}", mcp=mcp - ) - yield wu + ).as_workunit() # Set subtype - subtype_mcp = MetadataChangeProposalWrapper( - entityType="container", - changeType=ChangeTypeClass.UPSERT, + yield MetadataChangeProposalWrapper( entityUrn=f"{container_urn}", - # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), aspect=SubTypesClass(typeNames=sub_types), - ) - wu = MetadataWorkUnit( - id=f"container-subtypes-{name}-{container_urn}", mcp=subtype_mcp - ) - yield wu + ).as_workunit() if domain_urn: yield from add_domain_to_entity_wu( - entity_type="container", entity_urn=container_urn, domain_urn=domain_urn, ) @@ -299,39 +267,23 @@ def gen_containers( # Set database container parent_container_mcp = MetadataChangeProposalWrapper( - entityType="container", - changeType=ChangeTypeClass.UPSERT, entityUrn=f"{container_urn}", - # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), aspect=ContainerClass(container=parent_container_urn), - # aspect=ContainerKeyClass(guid=database_container_key.guid()) ) - wu = MetadataWorkUnit( - id=f"container-parent-container-{name}-{container_urn}-{parent_container_urn}", - mcp=parent_container_mcp, - ) - - yield wu + yield parent_container_mcp.as_workunit() def add_dataset_to_container( - # FIXME: Union requires two or more type arguments - container_key: KeyType, - dataset_urn: str, + container_key: KeyType, dataset_urn: str ) -> Iterable[MetadataWorkUnit]: container_urn = make_container_urn( guid=container_key.guid(), ) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, + yield MetadataChangeProposalWrapper( entityUrn=f"{dataset_urn}", aspect=ContainerClass(container=f"{container_urn}"), - # aspect=ContainerKeyClass(guid=schema_container_key.guid()) - ) - wu = MetadataWorkUnit(id=f"container-{container_urn}-to-{dataset_urn}", mcp=mcp) - yield wu + ).as_workunit() def add_entity_to_container( @@ -340,14 +292,11 @@ def add_entity_to_container( container_urn = make_container_urn( guid=container_key.guid(), ) - mcp = MetadataChangeProposalWrapper( + yield MetadataChangeProposalWrapper( entityType=entity_type, - changeType=ChangeTypeClass.UPSERT, entityUrn=entity_urn, aspect=ContainerClass(container=f"{container_urn}"), - ) - wu = MetadataWorkUnit(id=f"container-{container_urn}-to-{entity_urn}", mcp=mcp) - yield wu + ).as_workunit() def mcps_from_mce( @@ -355,8 +304,6 @@ def mcps_from_mce( ) -> Iterable[MetadataChangeProposalWrapper]: for aspect in mce.proposedSnapshot.aspects: yield MetadataChangeProposalWrapper( - entityType=guess_entity_type(mce.proposedSnapshot.urn), - changeType=ChangeTypeClass.UPSERT, entityUrn=mce.proposedSnapshot.urn, auditHeader=mce.auditHeader, aspect=aspect, diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index b1e6197fa19df..a78224a2aac57 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -51,7 +51,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws import s3_util from datahub.ingestion.source.aws.aws_common import AwsSourceConfig -from datahub.ingestion.source.aws.s3_util import make_s3_urn +from datahub.ingestion.source.aws.s3_util import is_s3_uri, make_s3_urn from datahub.ingestion.source.glue_profiling_config import GlueProfilingConfig from datahub.ingestion.source.state.checkpoint import Checkpoint from datahub.ingestion.source.state.sql_common_state import ( @@ -709,7 +709,7 @@ def get_lineage_if_enabled( ] = mce_builder.get_aspect_if_available(mce, DatasetPropertiesClass) if dataset_properties and "Location" in dataset_properties.customProperties: location = dataset_properties.customProperties["Location"] - if location.startswith("s3://"): + if is_s3_uri(location): s3_dataset_urn = make_s3_urn(location, self.source_config.env) if self.source_config.glue_s3_lineage_direction == "upstream": upstream_lineage = UpstreamLineageClass( @@ -930,12 +930,11 @@ def _gen_domain_urn(self, dataset_name: str) -> Optional[str]: return None def _get_domain_wu( - self, dataset_name: str, entity_urn: str, entity_type: str + self, dataset_name: str, entity_urn: str ) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(dataset_name) if domain_urn: wus = add_domain_to_entity_wu( - entity_type=entity_type, entity_urn=entity_urn, domain_urn=domain_urn, ) @@ -985,7 +984,6 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: yield from self._get_domain_wu( dataset_name=full_table_name, entity_urn=dataset_urn, - entity_type="dataset", ) yield from self.add_table_to_database_container( dataset_urn=dataset_urn, db_name=database_name diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 0b7f2a2c2d66f..c2b31081c645e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -800,12 +800,10 @@ def _get_domain_wu( self, dataset_name: str, entity_urn: str, - entity_type: str, ) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(dataset_name) if domain_urn: wus = add_domain_to_entity_wu( - entity_type=entity_type, entity_urn=entity_urn, domain_urn=domain_urn, ) @@ -963,7 +961,6 @@ def gen_dataset_workunits( yield from self._get_domain_wu( dataset_name=str(datahub_dataset_name), entity_urn=dataset_urn, - entity_type="dataset", ) def gen_lineage( diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index 4a25e5e3fceba..3bc8ecc98538c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -13,7 +13,7 @@ logger: logging.Logger = logging.getLogger(__name__) -@dataclass(frozen=True, eq=True) +@dataclass class BigqueryColumn(BaseColumn): field_path: str is_partition_column: bool diff --git a/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py b/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py index 8ec6fd81642af..ec34268b10083 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py +++ b/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py @@ -17,7 +17,9 @@ def __init__(self, tmp_dir: str, skip_known_host_verification: bool = True): self.skip_known_host_verification = skip_known_host_verification self.last_repo_cloned: Optional[git.Repo] = None - def clone(self, ssh_key: Optional[SecretStr], repo_url: str) -> Path: + def clone( + self, ssh_key: Optional[SecretStr], repo_url: str, branch: Optional[str] = None + ) -> Path: unique_dir = str(uuid4()) keys_dir = f"{self.tmp_dir}/{unique_dir}/keys" checkout_dir = f"{self.tmp_dir}/{unique_dir}/checkout" @@ -58,6 +60,11 @@ def clone(self, ssh_key: Optional[SecretStr], repo_url: str) -> Path: env=dict(GIT_SSH_COMMAND=git_ssh_cmd), ) logger.info("✅ Cloning complete!") + + if branch is not None: + logger.info(f"Checking out branch {branch}") + self.last_repo_cloned.git.checkout(branch) + return pathlib.Path(checkout_dir) def get_last_repo_cloned(self) -> Optional[git.Repo]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py b/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py index 824c5dfa5e623..d0c7f85a3e4c6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py +++ b/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py @@ -29,7 +29,6 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( - ChangeTypeClass, CorpGroupInfoClass, CorpUserInfoClass, GroupMembershipClass, @@ -296,10 +295,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: yield wu group_origin_mcp = MetadataChangeProposalWrapper( - entityType="corpGroup", entityUrn=datahub_corp_group_snapshot.urn, - changeType=ChangeTypeClass.UPSERT, - aspectName="origin", aspect=OriginClass(OriginTypeClass.EXTERNAL, "AZURE_AD"), ) group_origin_wu_id = f"group-origin-{group_count + 1 if self.config.mask_group_id else datahub_corp_group_snapshot.urn}" @@ -310,10 +306,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: yield group_origin_wu group_status_mcp = MetadataChangeProposalWrapper( - entityType="corpGroup", entityUrn=datahub_corp_group_snapshot.urn, - changeType=ChangeTypeClass.UPSERT, - aspectName="status", aspect=StatusClass(removed=False), ) group_status_wu_id = f"group-status-{group_count + 1 if self.config.mask_group_id else datahub_corp_group_snapshot.urn}" @@ -445,10 +438,7 @@ def ingest_ad_users( yield wu user_origin_mcp = MetadataChangeProposalWrapper( - entityType="corpuser", entityUrn=datahub_corp_user_snapshot.urn, - changeType=ChangeTypeClass.UPSERT, - aspectName="origin", aspect=OriginClass(OriginTypeClass.EXTERNAL, "AZURE_AD"), ) user_origin_wu_id = f"user-origin-{user_count + 1 if self.config.mask_user_id else datahub_corp_user_snapshot.urn}" @@ -457,10 +447,7 @@ def ingest_ad_users( yield user_origin_wu user_status_mcp = MetadataChangeProposalWrapper( - entityType="corpuser", entityUrn=datahub_corp_user_snapshot.urn, - changeType=ChangeTypeClass.UPSERT, - aspectName="status", aspect=StatusClass(removed=False), ) user_status_wu_id = f"user-status-{user_count + 1 if self.config.mask_user_id else datahub_corp_user_snapshot.urn}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py index 20c5818b0afb4..df6722bf8bcb5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py @@ -20,7 +20,6 @@ from datahub.configuration.kafka import KafkaConsumerConnectionConfig from datahub.configuration.source_common import DatasetSourceConfigBase from datahub.emitter.mce_builder import ( - DEFAULT_ENV, make_data_platform_urn, make_dataplatform_instance_urn, make_dataset_urn_with_platform_instance, @@ -72,8 +71,6 @@ class KafkaTopicConfigKeys(str, Enum): class KafkaSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigBase): - env: str = DEFAULT_ENV - # TODO: inline the connection config connection: KafkaConsumerConnectionConfig = KafkaConsumerConnectionConfig() topic_patterns: AllowDenyPattern = AllowDenyPattern(allow=[".*"], deny=["^_.*"]) @@ -312,7 +309,6 @@ def _extract_record( if domain_urn: wus = add_domain_to_entity_wu( - entity_type="dataset", entity_urn=dataset_urn, domain_urn=domain_urn, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 701d5eaa5738b..74f64247d73d2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -1434,6 +1434,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: checkout_dir = git_clone.clone( ssh_key=self.source_config.github_info.deploy_key, repo_url=self.source_config.github_info.repo_ssh_locator, + branch=self.source_config.github_info.branch_for_clone, ) self.reporter.git_clone_latency = datetime.now() - start_time self.source_config.base_folder = checkout_dir.resolve() @@ -1463,6 +1464,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) ), repo_url=p_ref.repo_ssh_locator, + branch=p_ref.branch_for_clone, ) p_ref = p_checkout_dir.resolve() diff --git a/metadata-ingestion/src/datahub/ingestion/source/openapi.py b/metadata-ingestion/src/datahub/ingestion/source/openapi.py index 05e4c0f6db0c2..62ca733c3fbe1 100755 --- a/metadata-ingestion/src/datahub/ingestion/source/openapi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/openapi.py @@ -102,11 +102,6 @@ def get_swagger(self) -> Dict: return sw_dict -# class ParserWarning(UserWarning): -# def __init__(self, message: str, key: str) -> None: -# self.message - - class ApiWorkUnit(MetadataWorkUnit): pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py index dc2f6550217dd..044a2fb551741 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py +++ b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py @@ -500,7 +500,6 @@ def _extract_record( if domain_urn: wus = add_domain_to_entity_wu( - entity_type="dataset", entity_urn=dataset_urn, domain_urn=domain_urn, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py index 69228e5f61b92..aaf254b0ee815 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py +++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py @@ -350,7 +350,7 @@ def get_domain_workunit( if domain_urn: yield from add_domain_to_entity_wu( - domain_urn=domain_urn, entity_type="dataset", entity_urn=datasetUrn + domain_urn=domain_urn, entity_urn=datasetUrn ) def get_platform_instance_workunit(self, datasetUrn: str) -> WorkUnit: diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py index 36c3c77b231e8..05f526a0603ce 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py @@ -36,6 +36,9 @@ class SnowflakeObjectDomain(str, Enum): EXTERNAL_TABLE = "external table" VIEW = "view" MATERIALIZED_VIEW = "materialized view" + DATABASE = "database" + SCHEMA = "schema" + COLUMN = "column" GENERIC_PERMISSION_ERROR_KEY = "permission-error" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 8b2c722254b81..ff2074e933d3c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -1,4 +1,5 @@ import logging +from enum import Enum from typing import Dict, Optional, cast from pydantic import Field, SecretStr, root_validator, validator @@ -19,6 +20,12 @@ logger = logging.Logger(__name__) +class TagOption(str, Enum): + with_lineage = "with_lineage" + without_lineage = "without_lineage" + skip = "skip" + + class SnowflakeV2Config( SnowflakeConfig, SnowflakeUsageConfig, @@ -53,6 +60,14 @@ class SnowflakeV2Config( default=None, description="Not supported" ) + extract_tags: TagOption = Field( + default=TagOption.skip, + description="""Optional. Allowed values are `without_lineage`, `with_lineage`, and `skip` (default). + `without_lineage` only extracts tags that have been applied directly to the given entity. + `with_lineage` extracts both directly applied and propagated tags, but will be significantly slower. + See the [Snowflake documentation](https://docs.snowflake.com/en/user-guide/object-tagging.html#tag-lineage) for information about tag lineage/propagation. """, + ) + classification: Optional[ClassificationConfig] = Field( default=None, description="For details, refer [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md).", @@ -76,6 +91,11 @@ def validate_include_column_lineage(cls, v, values): ) return v + tag_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="List of regex patterns for tags to include in ingestion. Only used if `extract_tags` is enabled.", + ) + @root_validator(pre=False) def validate_unsupported_configs(cls, values: Dict) -> Dict: value = values.get("provision_role") diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 0e378db317e91..34ed27a174218 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -105,6 +105,52 @@ def tables_for_schema(schema_name: str, db_name: Optional[str]) -> str: and table_type in ('BASE TABLE', 'EXTERNAL TABLE') order by table_schema, table_name""" + @staticmethod + def get_all_tags_on_object_with_propagation( + db_name: str, quoted_identifier: str, domain: str + ) -> str: + # https://docs.snowflake.com/en/sql-reference/functions/tag_references.html + return f""" + SELECT tag_database as "TAG_DATABASE", + tag_schema AS "TAG_SCHEMA", + tag_name AS "TAG_NAME", + tag_value AS "TAG_VALUE" + FROM table("{db_name}".information_schema.tag_references('{quoted_identifier}', '{domain}')); + """ + + @staticmethod + def get_all_tags_in_database_without_propagation(db_name: str) -> str: + # https://docs.snowflake.com/en/sql-reference/account-usage/tag_references.html + return f""" + SELECT tag_database as "TAG_DATABASE", + tag_schema AS "TAG_SCHEMA", + tag_name AS "TAG_NAME", + tag_value AS "TAG_VALUE", + object_database as "OBJECT_DATABASE", + object_schema AS "OBJECT_SCHEMA", + object_name AS "OBJECT_NAME", + column_name AS "COLUMN_NAME", + domain as "DOMAIN" + FROM snowflake.account_usage.tag_references + WHERE (object_database = '{db_name}' OR object_name = '{db_name}') + AND domain in ('DATABASE', 'SCHEMA', 'TABLE', 'COLUMN') + AND object_deleted IS NULL; + """ + + @staticmethod + def get_tags_on_columns_with_propagation( + db_name: str, quoted_table_identifier: str + ) -> str: + # https://docs.snowflake.com/en/sql-reference/functions/tag_references_all_columns.html + return f""" + SELECT tag_database as "TAG_DATABASE", + tag_schema AS "TAG_SCHEMA", + tag_name AS "TAG_NAME", + tag_value AS "TAG_VALUE", + column_name AS "COLUMN_NAME" + FROM table("{db_name}".information_schema.tag_references_all_columns('{quoted_table_identifier}', 'table')); + """ + # View definition is retrived in information_schema query only if role is owner of view. Hence this query is not used. # https://community.snowflake.com/s/article/Is-it-possible-to-see-the-view-definition-in-information-schema-views-from-a-non-owner-role @staticmethod diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index ff9f349cd9b67..5f4697c57a4f8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import MutableSet, Optional from datahub.ingestion.source.snowflake.constants import SnowflakeEdition from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport @@ -12,6 +12,7 @@ class SnowflakeV2Report(SnowflakeReport, SnowflakeUsageReport, ProfilingSqlRepor schemas_scanned: int = 0 databases_scanned: int = 0 + tags_scanned: int = 0 include_usage_stats: bool = False include_operational_stats: bool = False @@ -31,8 +32,16 @@ class SnowflakeV2Report(SnowflakeReport, SnowflakeUsageReport, ProfilingSqlRepor num_get_views_for_schema_queries: int = 0 num_get_columns_for_table_queries: int = 0 + # these will be non-zero if the user choses to enable the extract_tags = "with_lineage" option, which requires + # individual queries per object (database, schema, table) and an extra query per table to get the tags on the columns. + num_get_tags_for_object_queries: int = 0 + num_get_tags_on_columns_for_table_queries: int = 0 + rows_zero_objects_modified: int = 0 + _processed_tags: MutableSet[str] = set() + _scanned_tags: MutableSet[str] = set() + edition: Optional[SnowflakeEdition] = None def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: @@ -47,5 +56,21 @@ def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: self.schemas_scanned += 1 elif ent_type == "database": self.databases_scanned += 1 + elif ent_type == "tag": + # the same tag can be assigned to multiple objects, so we need + # some extra logic account for each tag only once. + if self._is_tag_scanned(name): + return + self._scanned_tags.add(name) + self.tags_scanned += 1 else: raise KeyError(f"Unknown entity {ent_type}.") + + def is_tag_processed(self, tag_name: str) -> bool: + return tag_name in self._processed_tags + + def _is_tag_scanned(self, tag_name: str) -> bool: + return tag_name in self._scanned_tags + + def report_tag_processed(self, tag_name: str) -> None: + self._processed_tags.add(tag_name) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 8031f9f5b5031..634ae6506b88f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict from dataclasses import dataclass, field from datetime import datetime from typing import Dict, List, Optional @@ -6,6 +7,7 @@ import pandas as pd from snowflake.connector import SnowflakeConnection +from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeQueryMixin from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView @@ -29,7 +31,21 @@ class SnowflakeFK: referred_column_names: List[str] -@dataclass(frozen=True, eq=True) +@dataclass +class SnowflakeTag: + database: str + schema: str + name: str + value: str + + def identifier(self) -> str: + return f"{self._id_prefix_as_str()}:{self.value}" + + def _id_prefix_as_str(self) -> str: + return f"{self.database}.{self.schema}.{self.name}" + + +@dataclass class SnowflakeColumn(BaseColumn): character_maximum_length: Optional[int] numeric_precision: Optional[int] @@ -61,12 +77,16 @@ class SnowflakeTable(BaseTable): pk: Optional[SnowflakePK] = None columns: List[SnowflakeColumn] = field(default_factory=list) foreign_keys: List[SnowflakeFK] = field(default_factory=list) + tags: Optional[List[SnowflakeTag]] = None + column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict) sample_data: Optional[pd.DataFrame] = None @dataclass class SnowflakeView(BaseView): columns: List[SnowflakeColumn] = field(default_factory=list) + tags: Optional[List[SnowflakeTag]] = None + column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict) @dataclass @@ -77,6 +97,7 @@ class SnowflakeSchema: comment: Optional[str] tables: List[SnowflakeTable] = field(default_factory=list) views: List[SnowflakeView] = field(default_factory=list) + tags: Optional[List[SnowflakeTag]] = None @dataclass @@ -86,6 +107,69 @@ class SnowflakeDatabase: comment: Optional[str] last_altered: Optional[datetime] = None schemas: List[SnowflakeSchema] = field(default_factory=list) + tags: Optional[List[SnowflakeTag]] = None + + +class _SnowflakeTagCache: + def __init__(self) -> None: + # self._database_tags[] = list of tags applied to database + self._database_tags: Dict[str, List[SnowflakeTag]] = defaultdict(list) + + # self._schema_tags[][] = list of tags applied to schema + self._schema_tags: Dict[str, Dict[str, List[SnowflakeTag]]] = defaultdict( + lambda: defaultdict(list) + ) + + # self._table_tags[][][] = list of tags applied to table + self._table_tags: Dict[ + str, Dict[str, Dict[str, List[SnowflakeTag]]] + ] = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + + # self._column_tags[][][][] = list of tags applied to column + self._column_tags: Dict[ + str, Dict[str, Dict[str, Dict[str, List[SnowflakeTag]]]] + ] = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + ) + + def add_database_tag(self, db_name: str, tag: SnowflakeTag) -> None: + self._database_tags[db_name].append(tag) + + def get_database_tags(self, db_name: str) -> List[SnowflakeTag]: + return self._database_tags[db_name] + + def add_schema_tag(self, schema_name: str, db_name: str, tag: SnowflakeTag) -> None: + self._schema_tags[db_name][schema_name].append(tag) + + def get_schema_tags(self, schema_name: str, db_name: str) -> List[SnowflakeTag]: + return self._schema_tags.get(db_name, {}).get(schema_name, []) + + def add_table_tag( + self, table_name: str, schema_name: str, db_name: str, tag: SnowflakeTag + ) -> None: + self._table_tags[db_name][schema_name][table_name].append(tag) + + def get_table_tags( + self, table_name: str, schema_name: str, db_name: str + ) -> List[SnowflakeTag]: + return self._table_tags[db_name][schema_name][table_name] + + def add_column_tag( + self, + column_name: str, + table_name: str, + schema_name: str, + db_name: str, + tag: SnowflakeTag, + ) -> None: + self._column_tags[db_name][schema_name][table_name][column_name].append(tag) + + def get_column_tags_for_table( + self, table_name: str, schema_name: str, db_name: str + ) -> Dict[str, List[SnowflakeTag]]: + return ( + self._column_tags.get(db_name, {}).get(schema_name, {}).get(table_name, {}) + ) class SnowflakeDataDictionary(SnowflakeQueryMixin): @@ -358,3 +442,101 @@ def get_fk_constraints_for_schema( constraints[row["fk_table_name"]].append(fk_constraints_map[row["fk_name"]]) return constraints + + def get_tags_for_database_without_propagation( + self, + db_name: str, + ) -> _SnowflakeTagCache: + cur = self.query( + SnowflakeQuery.get_all_tags_in_database_without_propagation(db_name) + ) + + tags = _SnowflakeTagCache() + + for tag in cur: + snowflake_tag = SnowflakeTag( + database=tag["TAG_DATABASE"], + schema=tag["TAG_SCHEMA"], + name=tag["TAG_NAME"], + value=tag["TAG_VALUE"], + ) + + # This is the name of the object, unless the object is a column, in which + # case the name is in the `COLUMN_NAME` field. + object_name = tag["OBJECT_NAME"] + # This will be null if the object is a database or schema + object_schema = tag["OBJECT_SCHEMA"] + # This will be null if the object is a database + object_database = tag["OBJECT_DATABASE"] + + domain = tag["DOMAIN"].lower() + if domain == SnowflakeObjectDomain.DATABASE: + tags.add_database_tag(object_name, snowflake_tag) + elif domain == SnowflakeObjectDomain.SCHEMA: + tags.add_schema_tag(object_name, object_database, snowflake_tag) + elif domain == SnowflakeObjectDomain.TABLE: # including views + tags.add_table_tag( + object_name, object_schema, object_database, snowflake_tag + ) + elif domain == SnowflakeObjectDomain.COLUMN: + column_name = tag["COLUMN_NAME"] + tags.add_column_tag( + column_name, + object_name, + object_schema, + object_database, + snowflake_tag, + ) + else: + # This should never happen. + self.logger.error(f"Encountered an unexpected domain: {domain}") + continue + + return tags + + def get_tags_for_object_with_propagation( + self, + domain: str, + quoted_identifier: str, + db_name: str, + ) -> List[SnowflakeTag]: + tags: List[SnowflakeTag] = [] + + cur = self.query( + SnowflakeQuery.get_all_tags_on_object_with_propagation( + db_name, quoted_identifier, domain + ), + ) + + for tag in cur: + tags.append( + SnowflakeTag( + database=tag["TAG_DATABASE"], + schema=tag["TAG_SCHEMA"], + name=tag["TAG_NAME"], + value=tag["TAG_VALUE"], + ) + ) + return tags + + def get_tags_on_columns_for_table( + self, quoted_table_name: str, db_name: str + ) -> Dict[str, List[SnowflakeTag]]: + tags: Dict[str, List[SnowflakeTag]] = defaultdict(list) + cur = self.query( + SnowflakeQuery.get_tags_on_columns_with_propagation( + db_name, quoted_table_name + ), + ) + + for tag in cur: + column_name = tag["COLUMN_NAME"] + snowflake_tag = SnowflakeTag( + database=tag["TAG_DATABASE"], + schema=tag["TAG_SCHEMA"], + name=tag["TAG_NAME"], + value=tag["TAG_VALUE"], + ) + tags[column_name].append(snowflake_tag) + + return tags diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_tag.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_tag.py new file mode 100644 index 0000000000000..e6b4ef1fd9607 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_tag.py @@ -0,0 +1,172 @@ +import logging +from typing import Dict, List, Optional + +from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain +from datahub.ingestion.source.snowflake.snowflake_config import ( + SnowflakeV2Config, + TagOption, +) +from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report +from datahub.ingestion.source.snowflake.snowflake_schema import ( + SnowflakeDataDictionary, + SnowflakeTag, + _SnowflakeTagCache, +) +from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeCommonMixin + +logger: logging.Logger = logging.getLogger(__name__) + + +class SnowflakeTagExtractor(SnowflakeCommonMixin): + def __init__( + self, + config: SnowflakeV2Config, + data_dictionary: SnowflakeDataDictionary, + report: SnowflakeV2Report, + ) -> None: + self.config = config + self.data_dictionary = data_dictionary + self.report = report + self.logger = logger + + self.tag_cache: Dict[str, _SnowflakeTagCache] = {} + + def _get_tags_on_object_without_propagation( + self, + domain: str, + db_name: str, + schema_name: Optional[str], + table_name: Optional[str], + ) -> List[SnowflakeTag]: + if db_name not in self.tag_cache: + self.tag_cache[ + db_name + ] = self.data_dictionary.get_tags_for_database_without_propagation(db_name) + + if domain == SnowflakeObjectDomain.DATABASE: + return self.tag_cache[db_name].get_database_tags(db_name) + elif domain == SnowflakeObjectDomain.SCHEMA: + assert schema_name is not None + tags = self.tag_cache[db_name].get_schema_tags(schema_name, db_name) + elif ( + domain == SnowflakeObjectDomain.TABLE + ): # Views belong to this domain as well. + assert schema_name is not None + assert table_name is not None + tags = self.tag_cache[db_name].get_table_tags( + table_name, schema_name, db_name + ) + else: + raise ValueError(f"Unknown domain {domain}") + return tags + + def _get_tags_on_object_with_propagation( + self, + domain: str, + db_name: str, + schema_name: Optional[str], + table_name: Optional[str], + ) -> List[SnowflakeTag]: + identifier = "" + if domain == SnowflakeObjectDomain.DATABASE: + identifier = self.get_quoted_identifier_for_database(db_name) + elif domain == SnowflakeObjectDomain.SCHEMA: + assert schema_name is not None + identifier = self.get_quoted_identifier_for_schema(db_name, schema_name) + elif ( + domain == SnowflakeObjectDomain.TABLE + ): # Views belong to this domain as well. + assert schema_name is not None + assert table_name is not None + identifier = self.get_quoted_identifier_for_table( + db_name, schema_name, table_name + ) + else: + raise ValueError(f"Unknown domain {domain}") + assert identifier + + self.report.num_get_tags_for_object_queries += 1 + tags = self.data_dictionary.get_tags_for_object_with_propagation( + domain=domain, quoted_identifier=identifier, db_name=db_name + ) + return tags + + def get_tags_on_object( + self, + domain: str, + db_name: str, + schema_name: Optional[str] = None, + table_name: Optional[str] = None, + ) -> List[SnowflakeTag]: + if self.config.extract_tags == TagOption.without_lineage: + tags = self._get_tags_on_object_without_propagation( + domain=domain, + db_name=db_name, + schema_name=schema_name, + table_name=table_name, + ) + + elif self.config.extract_tags == TagOption.with_lineage: + tags = self._get_tags_on_object_with_propagation( + domain=domain, + db_name=db_name, + schema_name=schema_name, + table_name=table_name, + ) + else: + tags = [] + + allowed_tags = self._filter_tags(tags) + + return allowed_tags if allowed_tags else [] + + def get_column_tags_for_table( + self, + table_name: str, + schema_name: str, + db_name: str, + ) -> Dict[str, List[SnowflakeTag]]: + temp_column_tags: Dict[str, List[SnowflakeTag]] = {} + if self.config.extract_tags == TagOption.without_lineage: + if db_name not in self.tag_cache: + self.tag_cache[ + db_name + ] = self.data_dictionary.get_tags_for_database_without_propagation( + db_name + ) + temp_column_tags = self.tag_cache[db_name].get_column_tags_for_table( + table_name, schema_name, db_name + ) + elif self.config.extract_tags == TagOption.with_lineage: + self.report.num_get_tags_on_columns_for_table_queries += 1 + temp_column_tags = self.data_dictionary.get_tags_on_columns_for_table( + quoted_table_name=self.get_quoted_identifier_for_table( + db_name, schema_name, table_name + ), + db_name=db_name, + ) + + column_tags: Dict[str, List[SnowflakeTag]] = {} + + for column_name in temp_column_tags: + tags = temp_column_tags[column_name] + allowed_tags = self._filter_tags(tags) + if allowed_tags: + column_tags[column_name] = allowed_tags + + return column_tags + + def _filter_tags( + self, tags: Optional[List[SnowflakeTag]] + ) -> Optional[List[SnowflakeTag]]: + if tags is None: + return tags + + allowed_tags = [] + for tag in tags: + tag_identifier = tag.identifier() + self.report.report_entity_scanned(tag_identifier, "tag") + if not self.config.tag_pattern.allowed(tag_identifier): + self.report.report_dropped(tag_identifier) + allowed_tags.append(tag) + return allowed_tags diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index 0d8f975ebda60..5f32a05680fa7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -158,6 +158,18 @@ def snowflake_identifier(self: SnowflakeCommonProtocol, identifier: str) -> str: return identifier.lower() return identifier + @staticmethod + def get_quoted_identifier_for_database(db_name): + return f'"{db_name}"' + + @staticmethod + def get_quoted_identifier_for_schema(db_name, schema_name): + return f'"{db_name}"."{schema_name}"' + + @staticmethod + def get_quoted_identifier_for_table(db_name, schema_name, table_name): + return f'"{db_name}"."{schema_name}"."{table_name}"' + def get_dataset_identifier( self: SnowflakeCommonProtocol, table_name: str, schema_name: str, db_name: str ) -> str: @@ -200,16 +212,10 @@ def wrap_aspect_as_workunit( aspectName: str, aspect: _Aspect, ) -> MetadataWorkUnit: - id = f"{aspectName}-for-{entityUrn}" - if "timestampMillis" in aspect._inner_dict: - id = f"{aspectName}-{aspect.timestampMillis}-for-{entityUrn}" # type: ignore - wu = MetadataWorkUnit( - id=id, - mcp=MetadataChangeProposalWrapper( - entityUrn=entityUrn, - aspect=aspect, - ), - ) + wu = MetadataChangeProposalWrapper( + entityUrn=entityUrn, + aspect=aspect, + ).as_workunit() self.report.report_workunit(wu) return wu diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 97b4b04700ddf..83b387039cec6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -15,6 +15,7 @@ make_dataset_urn_with_platform_instance, make_domain_urn, make_schema_field_urn, + make_tag_urn, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import ( @@ -49,7 +50,10 @@ SnowflakeEdition, SnowflakeObjectDomain, ) -from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config +from datahub.ingestion.source.snowflake.snowflake_config import ( + SnowflakeV2Config, + TagOption, +) from datahub.ingestion.source.snowflake.snowflake_lineage import ( SnowflakeLineageExtractor, ) @@ -64,8 +68,10 @@ SnowflakeQuery, SnowflakeSchema, SnowflakeTable, + SnowflakeTag, SnowflakeView, ) +from datahub.ingestion.source.snowflake.snowflake_tag import SnowflakeTagExtractor from datahub.ingestion.source.snowflake.snowflake_usage_v2 import ( SnowflakeUsageExtractor, ) @@ -90,8 +96,10 @@ StatefulIngestionSourceBase, ) from datahub.metadata.com.linkedin.pegasus2avro.common import ( + GlobalTags, Status, SubTypes, + TagAssociation, TimeStamp, ) from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( @@ -114,6 +122,7 @@ StringType, TimeType, ) +from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties from datahub.metadata.schema_classes import ChangeTypeClass, DataPlatformInstanceClass from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.time import datetime_to_ts_millis @@ -188,6 +197,11 @@ "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", supported=True, ) +@capability( + SourceCapability.TAGS, + "Optionally enabled via `extract_tags`", + supported=True, +) class SnowflakeV2Source( ClassificationMixin, SnowflakeQueryMixin, @@ -235,6 +249,10 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): # For usage stats self.usage_extractor = SnowflakeUsageExtractor(config, self.report) + self.tag_extractor = SnowflakeTagExtractor( + config, self.data_dictionary, self.report + ) + self.profiling_state_handler: Optional[ProfilingHandler] = None if self.config.store_last_profiling_timestamps: self.profiling_state_handler = ProfilingHandler( @@ -358,6 +376,7 @@ def query(query): _report[SourceCapability.CONTAINERS] = CapabilityReport( capable=True ) + _report[SourceCapability.TAGS] = CapabilityReport(capable=True) elif privilege.object_type in ( "TABLE", "VIEW", @@ -391,6 +410,8 @@ def query(query): _report[SourceCapability.USAGE_STATS] = CapabilityReport( capable=True ) + _report[SourceCapability.TAGS] = CapabilityReport(capable=True) + # If all capabilities supported, no need to continue if set(capabilities) == set(_report.keys()): break @@ -414,6 +435,7 @@ def query(query): SourceCapability.LINEAGE_COARSE: "Current role does not have permissions to snowflake account usage views", SourceCapability.LINEAGE_FINE: "Current role does not have permissions to snowflake account usage views", SourceCapability.USAGE_STATS: "Current role does not have permissions to snowflake account usage views", + SourceCapability.TAGS: "Either no tags have been applied to objects, or the current role does not have permission to access the objects or to snowflake account usage views ", } for c in capabilities: # type:ignore @@ -425,6 +447,7 @@ def query(query): SourceCapability.LINEAGE_COARSE, SourceCapability.LINEAGE_FINE, SourceCapability.USAGE_STATS, + SourceCapability.TAGS, ): failure_message = ( f"Current role {current_role} does not have permissions to use warehouse {connection_conf.warehouse}. Please check the grants associated with this role." @@ -471,6 +494,7 @@ def get_workunits(self) -> Iterable[WorkUnit]: for snowflake_db in databases: try: yield from self._process_database(snowflake_db) + except SnowflakePermissionError as e: # FIXME - This may break satetful ingestion if new tables than previous run are emitted above # and stateful ingestion is enabled @@ -627,11 +651,20 @@ def _process_database( ) return + if self.config.extract_tags != TagOption.skip: + snowflake_db.tags = self.tag_extractor.get_tags_on_object( + domain="database", db_name=db_name + ) + if self.config.include_technical_schema: yield from self.gen_database_containers(snowflake_db) self.fetch_schemas_for_database(snowflake_db, db_name) + if self.config.include_technical_schema and snowflake_db.tags: + for tag in snowflake_db.tags: + yield from self._process_tag(tag) + for snowflake_schema in snowflake_db.schemas: yield from self._process_schema(snowflake_schema, db_name) @@ -675,6 +708,12 @@ def _process_schema( return schema_name = snowflake_schema.name + + if self.config.extract_tags != TagOption.skip: + snowflake_schema.tags = self.tag_extractor.get_tags_on_object( + schema_name=schema_name, db_name=db_name, domain="schema" + ) + if self.config.include_technical_schema: yield from self.gen_schema_containers(snowflake_schema, db_name) @@ -692,6 +731,10 @@ def _process_schema( for view in snowflake_schema.views: yield from self._process_view(view, schema_name, db_name) + if self.config.include_technical_schema and snowflake_schema.tags: + for tag in snowflake_schema.tags: + yield from self._process_tag(tag) + if not snowflake_schema.views and not snowflake_schema.tables: self.report_warning( "No tables/views found in schema. If tables exist, please grant REFERENCES or SELECT permissions on them.", @@ -762,6 +805,22 @@ def _process_table( table, schema_name, db_name, dataset_name ) + if self.config.extract_tags != TagOption.skip: + table.tags = self.tag_extractor.get_tags_on_object( + table_name=table.name, + schema_name=schema_name, + db_name=db_name, + domain="table", + ) + + if self.config.include_technical_schema: + if table.tags: + for tag in table.tags: + yield from self._process_tag(tag) + for column_name in table.column_tags: + for tag in table.column_tags[column_name]: + yield from self._process_tag(tag) + yield from self.gen_dataset_workunits(table, schema_name, db_name) def fetch_sample_data_for_classification( @@ -817,6 +876,10 @@ def fetch_pk_for_table(self, table, schema_name, db_name, table_identifier): def fetch_columns_for_table(self, table, schema_name, db_name, table_identifier): try: table.columns = self.get_columns_for_table(table.name, schema_name, db_name) + if self.config.extract_tags != TagOption.skip: + table.column_tags = self.tag_extractor.get_column_tags_for_table( + table.name, schema_name, db_name + ) except Exception as e: logger.debug( f"Failed to get columns for table {table_identifier} due to error {e}", @@ -840,6 +903,10 @@ def _process_view( try: view.columns = self.get_columns_for_table(view.name, schema_name, db_name) + if self.config.extract_tags != TagOption.skip: + view.column_tags = self.tag_extractor.get_column_tags_for_table( + view.name, schema_name, db_name + ) except Exception as e: logger.debug( f"Failed to get columns for view {view_name} due to error {e}", @@ -847,8 +914,34 @@ def _process_view( ) self.report_warning("Failed to get columns for view", view_name) + if self.config.extract_tags != TagOption.skip: + view.tags = self.tag_extractor.get_tags_on_object( + table_name=view.name, + schema_name=schema_name, + db_name=db_name, + domain="table", + ) + + if self.config.include_technical_schema: + if view.tags: + for tag in view.tags: + yield from self._process_tag(tag) + for column_name in view.column_tags: + for tag in view.column_tags[column_name]: + yield from self._process_tag(tag) + yield from self.gen_dataset_workunits(view, schema_name, db_name) + def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: + tag_identifier = tag.identifier() + + if self.report.is_tag_processed(tag_identifier): + return + + self.report.report_tag_processed(tag_identifier) + + yield from self.gen_tag_workunits(tag) + def gen_dataset_workunits( self, table: Union[SnowflakeTable, SnowflakeView], @@ -905,9 +998,17 @@ def gen_dataset_workunits( yield from self._get_domain_wu( dataset_name=dataset_name, entity_urn=dataset_urn, - entity_type="dataset", ) + if table.tags: + tag_associations = [ + TagAssociation(tag=make_tag_urn(tag.identifier())) for tag in table.tags + ] + global_tags = GlobalTags(tag_associations) + yield self.wrap_aspect_as_workunit( + "dataset", dataset_urn, "globalTags", global_tags + ) + if ( isinstance(table, SnowflakeView) and cast(SnowflakeView, table).view_definition is not None @@ -951,6 +1052,21 @@ def get_dataset_properties(self, table, schema_name, db_name, dataset_name): else None, ) + def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: + tag_key = tag.identifier() + tag_urn = make_tag_urn(self.snowflake_identifier(tag_key)) + + tag_properties_aspect = TagProperties( + name=tag_key, + description=f"Represents the Snowflake tag `{tag._id_prefix_as_str()}` with value `{tag.value}`.", + ) + + self.stale_entity_removal_handler.add_entity_to_state("tag", tag_urn) + + yield self.wrap_aspect_as_workunit( + "tag", tag_urn, "tagProperties", tag_properties_aspect + ) + def get_schema_metadata( self, table: Union[SnowflakeTable, SnowflakeView], @@ -980,6 +1096,18 @@ def get_schema_metadata( isPartOfKey=col.name in table.pk.column_names if isinstance(table, SnowflakeTable) and table.pk is not None else None, + globalTags=GlobalTags( + [ + TagAssociation( + make_tag_urn( + self.snowflake_identifier(tag.identifier()) + ) + ) + for tag in table.column_tags[col.name] + ] + ) + if col.name in table.column_tags + else None, ) for col in table.columns ], @@ -1082,12 +1210,10 @@ def _get_domain_wu( self, dataset_name: str, entity_urn: str, - entity_type: str, ) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(dataset_name) if domain_urn: wus = add_domain_to_entity_wu( - entity_type=entity_type, entity_urn=entity_urn, domain_urn=domain_urn, ) @@ -1168,6 +1294,9 @@ def gen_database_containers( else int(database.created.timestamp() * 1000) if database.created is not None else None, + tags=[self.snowflake_identifier(tag.identifier()) for tag in database.tags] + if database.tags + else None, ) self.stale_entity_removal_handler.add_entity_to_state( @@ -1215,6 +1344,9 @@ def gen_schema_containers( else int(schema.created.timestamp() * 1000) if schema.created is not None else None, + tags=[self.snowflake_identifier(tag.identifier()) for tag in schema.tags] + if schema.tags + else None, ) for wu in container_workunits: diff --git a/metadata-ingestion/src/datahub/ingestion/source/source_registry.py b/metadata-ingestion/src/datahub/ingestion/source/source_registry.py index f2bfcd9ca9266..c3e61f58414f6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/source_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/source_registry.py @@ -1,5 +1,6 @@ import warnings +from datahub.configuration.common import ConfigurationWarning from datahub.ingestion.api.registry import PluginRegistry from datahub.ingestion.api.source import Source @@ -14,14 +15,16 @@ "snowflake-beta", "snowflake", lambda: warnings.warn( - UserWarning("source type snowflake-beta is deprecated, use snowflake instead") + "source type snowflake-beta is deprecated, use snowflake instead", + ConfigurationWarning, ), ) source_registry.register_alias( "bigquery-beta", "bigquery", lambda: warnings.warn( - UserWarning("source type bigquery-beta is deprecated, use bigquery instead") + "source type bigquery-beta is deprecated, use bigquery instead", + ConfigurationWarning, ), ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index b8f341cf27733..fc6c821bf90db 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -88,11 +88,13 @@ def get_sql_alchemy_url(self): SourceCapability.DATA_PROFILING, "Optionally enabled via configuration. Profiling uses sql queries on whole table which can be expensive operation.", ) +@capability(SourceCapability.LINEAGE_COARSE, "Supported for S3 tables") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") class AthenaSource(SQLAlchemySource): """ This plugin supports extracting the following metadata from Athena - Tables, schemas etc. + - Lineage for S3 tables. - Profiling when enabled. """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py index 0f0e11310e65c..875676aa0cc2a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py @@ -57,11 +57,6 @@ class SQLServerConfig(BasicSQLAlchemyConfig): description="database (catalog). If set to Null, all databases will be considered for ingestion.", ) - database_alias: Optional[str] = Field( - default=None, - description="Alias to apply to database when ingesting. Ignored when `database` is not set.", - ) - @pydantic.validator("uri_args") def passwords_match(cls, v, values, **kwargs): if values["use_odbc"] and "driver" not in v: @@ -203,7 +198,7 @@ def get_table_properties( ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: description, properties, location_urn = super().get_table_properties( inspector, schema, table - ) # type:Tuple[Optional[str], Dict[str, str], Optional[str]] + ) # Update description if available. db_name: str = self.get_db_name(inspector) description = self.table_descriptions.get( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py index c21177ebd9117..30c702b76c4b8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py @@ -1,3 +1,6 @@ +from collections import defaultdict +from typing import Dict, Iterable, List, Tuple, Union + # This import verifies that the dependencies are available. import psycopg2 # noqa: F401 import sqlalchemy.dialects.postgresql as custom_types @@ -8,9 +11,14 @@ # effects of the import. For more details, see here: # https://geoalchemy-2.readthedocs.io/en/latest/core_tutorial.html#reflecting-tables. from geoalchemy2 import Geometry # noqa: F401 +from pydantic import BaseModel from pydantic.fields import Field +from sqlalchemy import create_engine from datahub.configuration.common import AllowDenyPattern +from datahub.emitter import mce_builder +from datahub.emitter.mcp_builder import mcps_from_mce +from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, SupportStatus, @@ -19,9 +27,11 @@ platform_name, support_status, ) +from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.sql_common import ( BasicSQLAlchemyConfig, SQLAlchemySource, + SqlWorkUnit, register_custom_type, ) from datahub.metadata.com.linkedin.pegasus2avro.schema import ( @@ -36,10 +46,60 @@ register_custom_type(custom_types.HSTORE, MapTypeClass) +VIEW_LINEAGE_QUERY = """ +WITH RECURSIVE view_deps AS ( +SELECT DISTINCT dependent_ns.nspname as dependent_schema +, dependent_view.relname as dependent_view +, source_ns.nspname as source_schema +, source_table.relname as source_table +FROM pg_depend +JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid +JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid +JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid +JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace +JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace +WHERE NOT (dependent_ns.nspname = source_ns.nspname AND dependent_view.relname = source_table.relname) +UNION +SELECT DISTINCT dependent_ns.nspname as dependent_schema +, dependent_view.relname as dependent_view +, source_ns.nspname as source_schema +, source_table.relname as source_table +FROM pg_depend +JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid +JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid +JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid +JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace +JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace +INNER JOIN view_deps vd + ON vd.dependent_schema = source_ns.nspname + AND vd.dependent_view = source_table.relname + AND NOT (dependent_ns.nspname = vd.dependent_schema AND dependent_view.relname = vd.dependent_view) +) + + +SELECT source_table, source_schema, dependent_view, dependent_schema +FROM view_deps +WHERE NOT (source_schema = 'information_schema' OR source_schema = 'pg_catalog') +ORDER BY source_schema, source_table; +""" + + +class ViewLineageEntry(BaseModel): + # note that the order matches our query above + # so pydantic is able to parse the tuple using parse_obj + source_table: str + source_schema: str + dependent_view: str + dependent_schema: str + + class PostgresConfig(BasicSQLAlchemyConfig): # defaults scheme = Field(default="postgresql+psycopg2", description="database scheme") schema_pattern = Field(default=AllowDenyPattern(deny=["information_schema"])) + include_view_lineage = Field( + default=False, description="Include table lineage for views" + ) def get_identifier(self: BasicSQLAlchemyConfig, schema: str, table: str) -> str: regular = f"{schema}.{table}" @@ -56,6 +116,7 @@ def get_identifier(self: BasicSQLAlchemyConfig, schema: str, table: str) -> str: @capability(SourceCapability.DOMAINS, "Enabled by default") @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") +@capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration") class PostgresSource(SQLAlchemySource): """ This plugin extracts the following: @@ -67,10 +128,94 @@ class PostgresSource(SQLAlchemySource): - Table, row, and column statistics via optional SQL profiling """ - def __init__(self, config, ctx): + config: PostgresConfig + + def __init__(self, config: PostgresConfig, ctx: PipelineContext): super().__init__(config, ctx, "postgres") @classmethod def create(cls, config_dict, ctx): config = PostgresConfig.parse_obj(config_dict) return cls(config, ctx) + + def get_workunits(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: + yield from super().get_workunits() + + if self.config.include_view_lineage: + yield from self._get_view_lineage_workunits() + + def _get_view_lineage_elements(self) -> Dict[Tuple[str, str], List[str]]: + url = self.config.get_sql_alchemy_url() + engine = create_engine(url, **self.config.options) + + data: List[ViewLineageEntry] = [] + with engine.connect() as conn: + results = conn.execute(VIEW_LINEAGE_QUERY) + if results.returns_rows is False: + return {} + + for row in results: + data.append(ViewLineageEntry.parse_obj(row)) + + lineage_elements: Dict[Tuple[str, str], List[str]] = defaultdict(list) + # Loop over the lineages in the JSON data. + for lineage in data: + if not self.config.view_pattern.allowed(lineage.dependent_view): + self.report.report_dropped( + f"{lineage.dependent_schema}.{lineage.dependent_view}" + ) + continue + + if not self.config.schema_pattern.allowed(lineage.dependent_schema): + self.report.report_dropped( + f"{lineage.dependent_schema}.{lineage.dependent_view}" + ) + continue + + key = (lineage.dependent_view, lineage.dependent_schema) + # Append the source table to the list. + lineage_elements[key].append( + mce_builder.make_dataset_urn( + self.platform, + self.config.get_identifier( + lineage.source_schema, + lineage.source_table, + ), + self.config.env, + ) + ) + + return lineage_elements + + def _get_view_lineage_workunits(self) -> Iterable[MetadataWorkUnit]: + lineage_elements = self._get_view_lineage_elements() + + if not lineage_elements: + return None + + # Loop over the lineage elements dictionary. + for key, source_tables in lineage_elements.items(): + # Split the key into dependent view and dependent schema + dependent_view, dependent_schema = key + + # Construct a lineage object. + urn = mce_builder.make_dataset_urn( + self.platform, + self.config.get_identifier( + dependent_schema, + dependent_view, + ), + self.config.env, + ) + + # use the mce_builder to ensure that the change proposal inherits + # the correct defaults for auditHeader and systemMetadata + lineage_mce = mce_builder.make_lineage_mce( + source_tables, + urn, + ) + + for item in mcps_from_mce(lineage_mce): + wu = item.as_workunit() + self.report.report_workunit(wu) + yield wu diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index 718585b49283f..b37e2712e2625 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -527,7 +527,6 @@ def loop_tables( yield from self._get_domain_wu( dataset_name=dataset_name, entity_urn=dataset_urn, - entity_type="dataset", sql_config=sql_config, ) @@ -744,7 +743,6 @@ def loop_views( yield from self._get_domain_wu( dataset_name=dataset.dataset_name, entity_urn=dataset_urn, - entity_type="dataset", sql_config=sql_config, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 6db79f2fb6f69..8a63da437fe24 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -30,6 +30,7 @@ from sqlalchemy.types import TypeDecorator, TypeEngine from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated from datahub.emitter.mce_builder import ( make_container_urn, make_data_platform_urn, @@ -258,6 +259,11 @@ class SQLAlchemyConfig(StatefulIngestionConfigBase): default=True, description="Whether tables should be ingested." ) + include_table_location_lineage: bool = Field( + default=True, + description="If the source supports it, include table lineage to the underlying storage location.", + ) + profiling: GEProfilingConfig = GEProfilingConfig() # Custom Stateful Ingestion settings stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None @@ -303,15 +309,20 @@ class BasicSQLAlchemyConfig(SQLAlchemyConfig): description="URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", ) + _database_alias_deprecation = pydantic_field_deprecated( + "database_alias", + message="database_alias is deprecated. Use platform_instance instead.", + ) + def get_sql_alchemy_url(self, uri_opts: Optional[Dict[str, Any]] = None) -> str: if not ((self.host_port and self.scheme) or self.sqlalchemy_uri): raise ValueError("host_port and schema or connect_uri required.") return self.sqlalchemy_uri or make_sqlalchemy_uri( - self.scheme, # type: ignore + self.scheme, self.username, self.password.get_secret_value() if self.password is not None else None, - self.host_port, # type: ignore + self.host_port, self.database, uri_opts=uri_opts, ) @@ -734,13 +745,11 @@ def _get_domain_wu( self, dataset_name: str, entity_urn: str, - entity_type: str, sql_config: SQLAlchemyConfig, ) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(dataset_name) if domain_urn: wus = add_domain_to_entity_wu( - entity_type=entity_type, entity_urn=entity_urn, domain_urn=domain_urn, ) @@ -841,7 +850,7 @@ def _process_table( ) dataset_snapshot.aspects.append(dataset_properties) - if location_urn: + if self.config.include_table_location_lineage and location_urn: external_upstream_table = UpstreamClass( dataset=location_urn, type=DatasetLineageTypeClass.COPY, @@ -901,7 +910,6 @@ def _process_table( yield from self._get_domain_wu( dataset_name=dataset_name, entity_urn=dataset_urn, - entity_type="dataset", sql_config=sql_config, ) @@ -1185,7 +1193,6 @@ def _process_view( yield from self._get_domain_wu( dataset_name=dataset_name, entity_urn=dataset_urn, - entity_type="dataset", sql_config=sql_config, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic.py index 5acbb1983f9d4..3fbe8c40dff8c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic.py @@ -16,7 +16,7 @@ from datahub.ingestion.source.sql.sql_common import SQLAlchemyConfig, SQLAlchemySource -@dataclass(frozen=True, eq=True) +@dataclass class BaseColumn: name: str ordinal_position: int diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index 19e6c07e42f5b..2f6e84fa54dba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -1,14 +1,23 @@ -import re +import logging +import traceback +from collections import defaultdict +from dataclasses import dataclass from textwrap import dedent -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Set, Tuple, Union import pydantic from pydantic.class_validators import validator -from sqlalchemy import sql, util -from sqlalchemy.sql import sqltypes -from sqlalchemy.sql.sqltypes import TIME, TIMESTAMP, String -from sqlalchemy_vertica.base import VerticaDialect +from sqlalchemy import create_engine, sql +from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.exc import ProgrammingError +from datahub.configuration.common import AllowDenyPattern +from datahub.emitter.mce_builder import ( + dataset_urn_to_key, + make_dataset_urn_with_platform_instance, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_owner_to_entity_wu from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -18,254 +27,1315 @@ platform_name, support_status, ) +from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.sql_common import ( BasicSQLAlchemyConfig, + SQLAlchemyConfig, SQLAlchemySource, + SQLSourceReport, + SqlWorkUnit, + get_schema_metadata, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass +from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage +from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot +from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent +from datahub.metadata.schema_classes import ( + ChangeTypeClass, + DatasetLineageTypeClass, + DatasetPropertiesClass, + SubTypesClass, + UpstreamClass, + _Aspect, ) from datahub.utilities import config_clean +if TYPE_CHECKING: + from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest +MISSING_COLUMN_INFO = "missing column information" +logger: logging.Logger = logging.getLogger(__name__) + + +@dataclass +class VerticaSourceReport(SQLSourceReport): + projection_scanned: int = 0 + models_scanned: int = 0 + oauth_scanned: int = 0 + + def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: + """ + Entity could be a projection or a models or Oauth . + """ + + if ent_type == "projection": + self.projection_scanned += 1 + elif ent_type == "models": + self.models_scanned += 1 + elif ent_type == "oauth": + self.oauth_scanned += 1 + else: + super().report_entity_scanned(name, ent_type) + + +# Extended BasicSQLAlchemyConfig to config for projections,models and oauth metadata. +class VerticaConfig(BasicSQLAlchemyConfig): + models_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for ml models to filter in ingestion. ", + ) + include_projections: Optional[bool] = pydantic.Field( + default=True, description="Whether projections should be ingested." + ) + include_models: Optional[bool] = pydantic.Field( + default=True, description="Whether Models should be ingested." + ) + include_oauth: Optional[bool] = pydantic.Field( + default=True, description="Whether Oauth should be ingested." + ) + include_view_lineage: Optional[bool] = pydantic.Field( + default=True, + description="If the source supports it, include view lineage to the underlying storage location.", + ) + include_projection_lineage: Optional[bool] = pydantic.Field( + default=True, + description="If the source supports it, include view lineage to the underlying storage location.", + ) + + # defaults + scheme: str = pydantic.Field(default="vertica+vertica_python") + + @validator("host_port") + def clean_host_port(cls, v): + return config_clean.remove_protocol(v) + + +@platform_name("Vertica") +@config_class(VerticaConfig) +@support_status(SupportStatus.CERTIFIED) +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") +@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") +@capability( + SourceCapability.LINEAGE_COARSE, + "Enabled by default, can be disabled via configuration `include_view_lineage` and `include_projection_lineage`", +) +@capability( + SourceCapability.DELETION_DETECTION, + "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", + supported=True, +) +class VerticaSource(SQLAlchemySource): + def __init__(self, config: VerticaConfig, ctx: PipelineContext): + # self.platform = platform + super(VerticaSource, self).__init__(config, ctx, "vertica") + self.report: SQLSourceReport = VerticaSourceReport() + self.view_lineage_map: Optional[Dict[str, List[Tuple[str, str, str]]]] = None + self.projection_lineage_map: Optional[ + Dict[str, List[Tuple[str, str, str]]] + ] = None + self.config: VerticaConfig = config + + @classmethod + def create(cls, config_dict: Dict, ctx: PipelineContext) -> "VerticaSource": + config = VerticaConfig.parse_obj(config_dict) + return cls(config, ctx) + + def get_workunits(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: + sql_config = self.config + if logger.isEnabledFor(logging.DEBUG): + # If debug logging is enabled, we also want to echo each SQL query issued. + sql_config.options.setdefault("echo", True) + + # Extra default SQLAlchemy option for better connection pooling and threading. + # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow + if sql_config.profiling.enabled: + sql_config.options.setdefault( + "max_overflow", sql_config.profiling.max_workers + ) + + for inspector in self.get_inspectors(): + profiler = None + profile_requests: List["GEProfilerRequest"] = [] + if sql_config.profiling.enabled: + profiler = self.get_profiler_instance(inspector) + + db_name = self.get_db_name(inspector) + yield from self.gen_database_containers( + inspector=inspector, database=db_name + ) + + for schema in self.get_allowed_schemas(inspector, db_name): + self.add_information_for_schema(inspector, schema) + + yield from self.gen_schema_containers( + inspector=inspector, schema=schema, db_name=db_name + ) + + if sql_config.include_tables: + yield from self.loop_tables(inspector, schema, sql_config) + + if sql_config.include_views: + yield from self.loop_views(inspector, schema, sql_config) + if sql_config.include_projections: + yield from self.loop_projections(inspector, schema, sql_config) + if sql_config.include_models: + yield from self.loop_models(inspector, schema, sql_config) + + if profiler: + profile_requests += list( + self.loop_profiler_requests(inspector, schema, sql_config) + ) + + if profiler and profile_requests: + yield from self.loop_profiler( + profile_requests, profiler, platform=self.platform + ) + + oauth_schema = "Entities" + if sql_config.include_oauth: + yield from self.loop_oauth(inspector, oauth_schema, sql_config) + + # Clean up stale entities. + yield from self.stale_entity_removal_handler.gen_removed_entity_workunits() + + def get_database_properties( + self, inspector: Inspector, database: str + ) -> Optional[Dict[str, str]]: + try: + custom_properties = inspector._get_database_properties(database) # type: ignore + return custom_properties + except Exception as ex: + self.report.report_failure( + f"{database}", f"unable to get extra_properties : {ex}" + ) + return None + + def get_schema_properties( + self, inspector: Inspector, database: str, schema: str + ) -> Optional[Dict[str, str]]: + try: + custom_properties = inspector._get_schema_properties(schema) # type: ignore + return custom_properties + except Exception as ex: + self.report.report_failure( + f"{database}.{schema}", f"unable to get extra_properties : {ex}" + ) + return None + + def _process_table( + self, + dataset_name: str, + inspector: Inspector, + schema: str, + table: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + table_owner = self._get_owner_information(table, "table") + yield from add_owner_to_entity_wu( + entity_type="dataset", + entity_urn=dataset_urn, + owner_urn=f"urn:li:corpuser:{table_owner}", + ) + yield from super()._process_table( + dataset_name, inspector, schema, table, sql_config + ) + + def loop_views( + self, + inspector: Inspector, + schema: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + try: + for view in inspector.get_view_names(schema): + schema, view = self.standardize_schema_table_names( + schema=schema, entity=view + ) + dataset_name = self.get_identifier( + schema=schema, entity=view, inspector=inspector + ) + dataset_name = self.normalise_dataset_name(dataset_name) + + self.report.report_entity_scanned(dataset_name, ent_type="view") + + if not sql_config.view_pattern.allowed(dataset_name): + self.report.report_dropped(dataset_name) + continue + + try: + yield from self._process_view( + dataset_name=dataset_name, + inspector=inspector, + schema=schema, + view=view, + sql_config=sql_config, + ) + except Exception as e: + logger.warning( + f"Unable to ingest view {schema}.{view} due to an exception.\n {traceback.format_exc()}" + ) + self.report.report_warning( + f"{schema}.{view}", f"Ingestion error: {e}" + ) + if sql_config.include_view_lineage: # type: ignore + try: + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, aspects=[StatusClass(removed=False)] + ) + lineage_info = self._get_upstream_lineage_info( + dataset_urn, view + ) + if lineage_info is not None: + # Emit the lineage work unit + # upstream_column_props = [] + + upstream_lineage = lineage_info + lineage_mcpw = MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_snapshot.urn, + aspectName="upstreamLineage", + aspect=upstream_lineage, + ) + lineage_wu = MetadataWorkUnit( + id=f"{self.platform}-{lineage_mcpw.entityUrn}-{lineage_mcpw.aspectName}", + mcp=lineage_mcpw, + ) + self.report.report_workunit(lineage_wu) + yield lineage_wu + + except Exception as e: + logger.warning( + f"Unable to get lineage of view {schema}.{view} due to an exception.\n {traceback.format_exc()}" + ) + self.report.report_warning( + f"{schema}.{view}", f"Ingestion error: {e}" + ) + except Exception as e: + self.report.report_failure(f"{schema}", f"Views error: {e}") + + def _process_view( + self, + dataset_name: str, + inspector: Inspector, + schema: str, + view: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + """ + This function is used for performing operation and gets data for every view inside a schema + + Args: + dataset_name (str) + inspector (Inspector) + schema (str): schame name + view (str): name of the view to inspect + sql_config (SQLAlchemyConfig) + table_tags (Dict[str, str], optional) Defaults to dict(). + + Returns: + Iterable[Union[SqlWorkUnit, MetadataWorkUnit]] + + Yields: + Iterator[Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]] + """ + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + view_owner = self._get_owner_information(view, "view") + yield from add_owner_to_entity_wu( + entity_type="dataset", + entity_urn=dataset_urn, + owner_urn=f"urn:li:corpuser:{view_owner}", + ) + yield from super()._process_view( + dataset_name, inspector, schema, view, sql_config + ) + + def loop_projections( + self, + inspector: Inspector, + schema: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + """ + this function loop through all the projection in the given schema. + projection is just like table is introduced by vertica db to store data + a super projection is automatically created when a table is created + + Args: + inspector (Inspector): inspector obj from reflection + schema (str): schema name + sql_config (SQLAlchemyConfig): config + + Returns: + Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: [description] + + Yields: + Iterator[Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]]: [description] + """ + projections_seen: Set[str] = set() + + try: + # table_tags = self.get_extra_tags(inspector, schema, "projection") + for projection in inspector.get_projection_names(schema): # type: ignore + + schema, projection = self.standardize_schema_table_names( + schema=schema, entity=projection + ) + dataset_name = self.get_identifier( + schema=schema, entity=projection, inspector=inspector + ) + dataset_name = self.normalise_dataset_name(dataset_name) + if dataset_name not in projections_seen: + projections_seen.add(dataset_name) + else: + logger.debug("has already been seen, skipping... %s", dataset_name) + continue + self.report.report_entity_scanned(dataset_name, ent_type="projection") + if not sql_config.table_pattern.allowed(dataset_name): + self.report.report_dropped(dataset_name) + continue + try: + yield from self._process_projections( + dataset_name, inspector, schema, projection, sql_config + ) + except Exception as ex: + logger.warning( + f"Unable to ingest {schema}.{projection} due to an exception %s", + ex, + ) + self.report.report_warning( + f"{schema}.{projection}", f"Ingestion error: {ex}" + ) + if sql_config.include_projection_lineage: # type: ignore + + try: + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[StatusClass(removed=False)], + ) + lineage_info = self._get_upstream_lineage_info_projection( + dataset_urn, projection + ) + + if lineage_info is not None: + # Emit the lineage work unit + + upstream_lineage = lineage_info + + lineage_mcpw = MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_snapshot.urn, + aspectName="upstreamLineage", + aspect=upstream_lineage, + ) + + lineage_wu = MetadataWorkUnit( + id=f"{self.platform}-{lineage_mcpw.entityUrn}-{lineage_mcpw.aspectName}", + mcp=lineage_mcpw, + ) + + self.report.report_workunit(lineage_wu) + yield lineage_wu + + except Exception as ex: + logger.warning( + f"Unable to get lineage of Projection {schema}.{projection} due to an exception %s", + ex, + ) + self.report.report_warning( + f"{schema}.{projection}", f"Ingestion error: {ex}" + ) + except Exception as ex: + self.report.report_failure(f"{schema}", f"Projection error: {ex}") + + def _process_projections( + self, + dataset_name: str, + inspector: Inspector, + schema: str, + projection: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + columns = self._get_columns(dataset_name, inspector, schema, projection) + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[StatusClass(removed=False)], + ) + # Add table to the checkpoint state + self.stale_entity_removal_handler.add_entity_to_state("table", dataset_urn) + description, properties, location_urn = self.get_projection_properties( + inspector, schema, projection + ) + + # Tablename might be different from the real table if we ran some normalisation ont it. + # Getting normalized table name from the dataset_name + # Table is the last item in the dataset name + normalised_table = projection + splits = dataset_name.split(".") + if splits: + normalised_table = splits[-1] + if properties and normalised_table != projection: + properties["original_table_name"] = projection + + dataset_properties = DatasetPropertiesClass( + name=normalised_table, + description=description, + customProperties=properties, + ) + dataset_snapshot.aspects.append(dataset_properties) + + if location_urn: + external_upstream_table = UpstreamClass( + dataset=location_urn, + type=DatasetLineageTypeClass.COPY, + ) + lineage_mcpw = MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_snapshot.urn, + aspectName="upstreamLineage", + aspect=UpstreamLineage(upstreams=[external_upstream_table]), + ) + lineage_wu = MetadataWorkUnit( + id=f"{self.platform}-{lineage_mcpw.entityUrn}-{lineage_mcpw.aspectName}", + mcp=lineage_mcpw, + ) + self.report.report_workunit(lineage_wu) + yield lineage_wu + + projection_owner = self._get_owner_information(projection, "projection") + yield from add_owner_to_entity_wu( + entity_type="dataset", + entity_urn=dataset_urn, + owner_urn=f"urn:li:corpuser:{projection_owner}", + ) + # extra_tags = self.get_extra_tags(inspector, schema, projection) + pk_constraints: dict = inspector.get_pk_constraint(projection, schema) + foreign_keys = self._get_foreign_keys( + dataset_urn, inspector, schema, projection + ) + schema_fields = self.get_schema_fields(dataset_name, columns, pk_constraints) + schema_metadata = get_schema_metadata( + self.report, + dataset_name, + self.platform, + columns, + pk_constraints, + foreign_keys, + schema_fields, + ) + dataset_snapshot.aspects.append(schema_metadata) + db_name = self.get_db_name(inspector) + yield from self.add_table_to_schema_container(dataset_urn, db_name, schema) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + wu = SqlWorkUnit(id=dataset_name, mce=mce) + self.report.report_workunit(wu) + yield wu + dpi_aspect = self.get_dataplatform_instance_aspect(dataset_urn=dataset_urn) + if dpi_aspect: + yield dpi_aspect + subtypes_aspect = MetadataWorkUnit( + id=f"{dataset_name}-subtypes", + mcp=MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_urn, + aspectName="subTypes", + aspect=SubTypesClass(typeNames=["Projections"]), + ), + ) + self.report.report_workunit(subtypes_aspect) + yield subtypes_aspect + + yield from self._get_domain_wu( + dataset_name=dataset_name, + entity_urn=dataset_urn, + sql_config=sql_config, + ) + + def get_projection_properties( + self, inspector: Inspector, schema: str, projection: str + ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: + """ + Returns projection related metadata information to show in properties tab + eg. projection type like super, segmented etc + partition key, segmentation and so on + + Args: + inspector (Inspector): inspector obj from reflection engine + schema (str): schema name + projection (str): projection name + Returns: + Tuple[Optional[str], Dict[str, str], Optional[str]]: [description] + """ + description: Optional[str] = None + properties: Dict[str, str] = {} + # The location cannot be fetched generically, but subclasses may override + # this method and provide a location. + location: Optional[str] = None + try: + # SQLAlchemy stubs are incomplete and missing this method. + # PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223. + projection_info: dict = inspector.get_projection_comment(projection, schema) # type: ignore + except NotImplementedError: + return description, properties, location + except ProgrammingError as error: + logger.debug( + f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and table {properties} %s", + error, + ) + projection_info: dict = inspector.get_projection_comment(properties, f'"{schema}"') # type: ignore + description = projection_info.get("text") + if isinstance(description, tuple): + # Handling for value type tuple which is coming for dialect 'db2+ibm_db' + description = projection_info["text"][0] + # The "properties" field is a non-standard addition to SQLAlchemy's interface. + properties = projection_info.get("properties", {}) + return description, properties, location + + def loop_models( + self, + inspector: Inspector, + schema: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + """ + This function is for iterating over the ml models in vertica db + + Args: + inspector (Inspector) : inspector obj from reflection engine + schema (str): schema name + sql_config (SQLAlchemyConfig): config + + Returns: + Iterable[Union[SqlWorkUnit, MetadataWorkUnit]] + + Yields: + Iterator[Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]]: + """ + models_seen: Set[str] = set() + try: + + for models in inspector.get_models_names(schema): # type: ignore + + schema, models = self.standardize_schema_table_names( + schema=schema, entity=models + ) + dataset_name = self.get_identifier( + schema="Entities", entity=models, inspector=inspector + ) + + dataset_name = self.normalise_dataset_name(dataset_name) + if dataset_name not in models_seen: + models_seen.add(dataset_name) + else: + logger.debug("has already been seen, skipping... %s", dataset_name) + continue + self.report.report_entity_scanned(dataset_name, ent_type="models") + if not sql_config.table_pattern.allowed(dataset_name): + self.report.report_dropped(dataset_name) + continue + try: + yield from self._process_models( + dataset_name, + inspector, + schema, + models, + sql_config, + ) + except Exception as error: + logger.warning( + f"Unable to ingest {schema}.{models} due to an exception. %s {traceback.format_exc()}" + ) + self.report.report_warning( + f"{schema}.{models}", f"Ingestion error: {error}" + ) + except Exception as error: + self.report.report_failure(f"{schema}", f"Model error: {error}") + + def _process_models( + self, + dataset_name: str, + inspector: Inspector, + schema: str, + table: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + """ + To fetch ml models related information of ml_model from vertica db + Args: + dataset_name (str): dataset name + inspector (Inspector): inspector obj from reflection + schema (str): schema name entity + table (str): name of ml model + sql_config (SQLAlchemyConfig) -class UUID(String): - """The SQL UUID type.""" + Returns: + Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: [description] + """ + columns: List[Dict[Any, Any]] = [] + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[StatusClass(removed=False)], + ) + # Add table to the checkpoint state + self.stale_entity_removal_handler.add_entity_to_state("model", dataset_urn) + description, properties, location = self.get_model_properties( + inspector, schema, table + ) + # Tablename might be different from the real table if we ran some normalisation ont it. + # Getting normalized table name from the dataset_name + # Table is the last item in the dataset name + + normalised_table = table + splits = dataset_name.split(".") + if splits: + normalised_table = splits[-1] + if properties and normalised_table != table: + properties["original_table_name"] = table + dataset_properties = DatasetPropertiesClass( + name=normalised_table, + description=description, + customProperties=properties, + ) + + dataset_snapshot.aspects.append(dataset_properties) + + schema_fields = self.get_schema_fields(dataset_name, columns) + + schema_metadata = get_schema_metadata( + self.report, + dataset_name, + self.platform, + columns, + schema_fields, # type: ignore + ) + + dataset_snapshot.aspects.append(schema_metadata) + db_name = self.get_db_name(inspector) + + yield from self.add_table_to_schema_container(dataset_urn, db_name, schema) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + wu = SqlWorkUnit(id=dataset_name, mce=mce) + self.report.report_workunit(wu) + yield wu + dpi_aspect = self.get_dataplatform_instance_aspect(dataset_urn=dataset_urn) + if dpi_aspect: + yield dpi_aspect + subtypes_aspect = MetadataWorkUnit( + id=f"{dataset_name}-subtypes", + mcp=MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_urn, + aspectName="subTypes", + aspect=SubTypesClass(typeNames=["ML Models"]), + ), + ) + self.report.report_workunit(subtypes_aspect) + yield subtypes_aspect + yield from self._get_domain_wu( + dataset_name=dataset_name, + entity_urn=dataset_urn, + sql_config=sql_config, + ) - __visit_name__ = "UUID" + def get_model_properties( + self, inspector: Inspector, schema: str, model: str + ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: + """ + Returns ml models related metadata information to show in properties tab + eg. ml model attribute and ml model specification information. + + Args: + inspector (Inspector): inspector obj from reflection engine + schema (str): schema name + model (str): ml model name + Returns: + Tuple[Optional[str], Dict[str, str], Optional[str]] + """ + description: Optional[str] = None + properties: Dict[str, str] = {} + # The location cannot be fetched generically, but subclasses may override + # this method and provide a location. + location: Optional[str] = None + try: + table_info: dict = inspector.get_model_comment(model, schema) # type: ignore + except NotImplementedError: + return description, properties, location + except ProgrammingError as error: + logger.debug( + f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and {model} %s", + error, + ) + table_info: dict = inspector.get_model_comment(model, f'"{schema}"') # type: ignore + description = table_info.get("text") + if type(description) is tuple: + # Handling for value type tuple which is coming for dialect 'db2+ibm_db' + description = table_info["text"][0] + # The "properties" field is a non-standard addition to SQLAlchemy's interface. + properties = table_info.get("properties", {}) + return description, properties, location + + def loop_oauth( + self, + inspector: Inspector, + schema: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + """ + This function is for iterating over the oauth in vertica db + Args: + inspector (Inspector): + schema (str): schema name + sql_config (SQLAlchemyConfig): configuration + + Returns: + Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: [description] + """ + oauth_seen: Set[str] = set() + try: + + for oauth in inspector.get_Oauth_names(schema): # type: ignore + + schema, oauth = self.standardize_schema_table_names( + schema=schema, entity=oauth + ) + dataset_name = self.get_identifier( + schema=schema, entity=oauth, inspector=inspector + ) + + dataset_name = self.normalise_dataset_name(dataset_name) + if dataset_name not in oauth_seen: + oauth_seen.add(dataset_name) + else: + logger.debug("has already been seen, skipping %s", dataset_name) + continue + self.report.report_entity_scanned(dataset_name, ent_type="oauth") + if not sql_config.table_pattern.allowed(dataset_name): + self.report.report_dropped(dataset_name) + continue + try: + yield from self._process_oauth( + dataset_name, + inspector, + schema, + oauth, + sql_config, + ) + except Exception as error: + self.report.report_warning( + f"{schema}.{oauth}", f"Ingestion error: {error}" + ) + except Exception as error: + self.report.report_failure(f"{schema}", f"oauth error: {error}") + + def _process_oauth( + self, + dataset_name: str, + inspector: Inspector, + schema: str, + oauth: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: + """ + To fetch oauth related metadata of oauth from vertica db + Args: + dataset_name (str): dataset name + inspector (Inspector): inspector object from reflection + schema (str): schema name + Oauth (str): oauth name + sql_config (SQLAlchemyConfig): configuration + Returns: + Iterable[Union[SqlWorkUnit, MetadataWorkUnit]] + """ + columns: List[Dict[Any, Any]] = [] + dataset_urn = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[StatusClass(removed=False)], + ) + # Add table to the checkpoint state + self.stale_entity_removal_handler.add_entity_to_state("oauth", dataset_urn) + description, properties, location_urn = self.get_oauth_properties( + inspector, schema, oauth + ) + # Tablename might be different from the real table if we ran some normalisation ont it. + # Getting normalized table name from the dataset_name + # Table is the last item in the dataset name + normalised_table = oauth + splits = dataset_name.split(".") + if splits: + normalised_table = splits[-1] + if properties and normalised_table != oauth: + properties["original_table_name"] = oauth + dataset_properties = DatasetPropertiesClass( + name=normalised_table, + description=description, + customProperties=properties, + ) + dataset_snapshot.aspects.append(dataset_properties) + schema_fields = self.get_schema_fields(dataset_name, columns) + schema_metadata = get_schema_metadata( + self.report, + dataset_name, + self.platform, + columns, + schema_fields, # type: ignore + ) + dataset_snapshot.aspects.append(schema_metadata) + db_name = self.get_db_name(inspector) -class TIMESTAMP_WITH_PRECISION(TIMESTAMP): - """The SQL TIMESTAMP With Precision type. + yield from self.add_table_to_schema_container(dataset_urn, db_name, schema) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + wu = SqlWorkUnit(id=dataset_name, mce=mce) + self.report.report_workunit(wu) + yield wu + dpi_aspect = self.get_dataplatform_instance_aspect(dataset_urn=dataset_urn) + if dpi_aspect: + yield dpi_aspect + subtypes_aspect = MetadataWorkUnit( + id=f"{dataset_name}-subtypes", + mcp=MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_urn, + aspectName="subTypes", + aspect=SubTypesClass(typeNames=["oauth"]), + ), + ) + self.report.report_workunit(subtypes_aspect) + yield subtypes_aspect - Since Vertica supports precision values for timestamp this allows ingestion - of timestamp fields with precision values. - PS: THIS DATA IS CURRENTLY UNUSED, IT JUST FIXES INGESTION PROBLEMS - TODO: Should research the possibility of reflecting the precision in the schema + yield from self._get_domain_wu( + dataset_name=dataset_name, + entity_urn=dataset_urn, + sql_config=sql_config, + ) - """ + def get_oauth_properties( + self, inspector: Inspector, schema: str, model: str + ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: + """ + Returns oauth related metadata information to show in properties tab + eg. is_auth_enabled , auth_priority and all auth related info. - __visit_name__ = "TIMESTAMP" + Args: + inspector (Inspector): inspector obj from reflection engine + schema (str): schema name + model (str): ml model name + Returns: + Tuple[Optional[str], Dict[str, str], Optional[str]] + """ + description: Optional[str] = None + properties: Dict[str, str] = {} + # The location cannot be fetched generically, but subclasses may override + # this method and provide a location. + location: Optional[str] = None + try: - def __init__(self, timezone=False, precision=None): - """Construct a new :class:`_types.TIMESTAMP_WITH_PRECISION`. + table_info: dict = inspector.get_oauth_comment(model, schema) # type: ignore + except NotImplementedError: + return description, properties, location + except ProgrammingError as error: + logger.debug( + f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and oauth {model} %s", + error, + ) + table_info: dict = inspector.get_oauth_comment(model, f'"{schema}"') # type: ignore + description = table_info.get("text") + if isinstance(description, tuple): + # Handling for value type tuple which is coming for dialect 'db2+ibm_db' + description = table_info["text"][0] + # The "properties" field is a non-standard addition to SQLAlchemy's interface. + properties = table_info.get("properties", {}) + return description, properties, location - :param timezone: boolean. Indicates that the TIMESTAMP type should - enable timezone support, if available on the target database. - On a per-dialect basis is similar to "TIMESTAMP WITH TIMEZONE". - If the target database does not support timezones, this flag is - ignored. - :param precision: integer. Indicates the PRECISION field when provided + def loop_profiler_requests( + self, + inspector: Inspector, + schema: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable["GEProfilerRequest"]: + """Function is used for collecting profiling related information for every projections + inside an schema. + Args: schema: schema name """ - super(TIMESTAMP, self).__init__(timezone=timezone) - self.precision = precision + from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest + tables_seen: Set[str] = set() + profile_candidates = None # Default value if profile candidates not available. + yield from super().loop_profiler_requests(inspector, schema, sql_config) + for projection in inspector.get_projection_names(schema): # type: ignore -def TIMESTAMP_WITH_TIMEZONE(*args, **kwargs): - kwargs["timezone"] = True - return TIMESTAMP_WITH_PRECISION(*args, **kwargs) + schema, projection = self.standardize_schema_table_names( + schema=schema, entity=projection + ) + dataset_name = self.get_identifier( + schema=schema, entity=projection, inspector=inspector + ) + if not self.is_dataset_eligible_for_profiling( + dataset_name, sql_config, inspector, profile_candidates + ): + if self.config.profiling.report_dropped_profiles: + self.report.report_dropped(f"profile of {dataset_name}") + continue + dataset_name = self.normalise_dataset_name(dataset_name) + if dataset_name not in tables_seen: + tables_seen.add(dataset_name) + else: + logger.debug(f"{dataset_name} has already been seen, skipping...") + continue + missing_column_info_warn = self.report.warnings.get(MISSING_COLUMN_INFO) + if ( + missing_column_info_warn is not None + and dataset_name in missing_column_info_warn + ): + continue + (partition, custom_sql) = self.generate_partition_profiler_query( + schema, projection, self.config.profiling.partition_datetime + ) + if partition is None and self.is_table_partitioned( + database=None, schema=schema, table=projection + ): + self.report.report_warning( + "profile skipped as partitioned table is empty or partition id was invalid", + dataset_name, + ) + continue + if ( + partition is not None + and not self.config.profiling.partition_profiling_enabled + ): + logger.debug( + f"{dataset_name} and partition {partition} is skipped because profiling.partition_profiling_enabled property is disabled" + ) + continue + self.report.report_entity_profiled(dataset_name) + logger.debug( + f"Preparing profiling request for {schema}, {projection}, {partition}" + ) -def TIME_WITH_TIMEZONE(*args, **kwargs): - kwargs["timezone"] = True - return TIME(*args, **kwargs) + yield GEProfilerRequest( + pretty_name=dataset_name, + batch_kwargs=self.prepare_profiler_args( + inspector=inspector, + schema=schema, + table=projection, + partition=partition, + custom_sql=custom_sql, + ), + ) + def _get_upstream_lineage_info( + self, dataset_urn: str, view: str + ) -> Optional[_Aspect]: -def get_view_definition(self, connection, view_name, schema=None, **kw): - if schema is not None: - schema_condition = "lower(table_schema) = '%(schema)s'" % { - "schema": schema.lower() - } - else: - schema_condition = "1" + dataset_key = dataset_urn_to_key(dataset_urn) + if dataset_key is None: + logger.warning(f"Invalid dataset urn {dataset_urn}. Could not get key!") + return None - view_def = connection.scalar( - sql.text( + self._populate_view_lineage(view) + dataset_name = dataset_key.name + lineage = self.view_lineage_map[dataset_name] # type: ignore + + if not (lineage): + logger.debug(f"No lineage found for {dataset_name}") + return None + upstream_tables: List[UpstreamClass] = [] + + for lineage_entry in lineage: + # Update the view-lineage + upstream_table_name = lineage_entry[0] + + upstream_table = UpstreamClass( + dataset=make_dataset_urn_with_platform_instance( + self.platform, + upstream_table_name, + self.config.platform_instance, + self.config.env, + ), + type=DatasetLineageTypeClass.TRANSFORMED, + ) + upstream_tables.append(upstream_table) + + if upstream_tables: + + logger.debug( + f" lineage of '{dataset_name}': {[u.dataset for u in upstream_tables]}" + ) + + return UpstreamLineage(upstreams=upstream_tables) + + return None + + def _populate_view_lineage(self, view: str) -> None: + """Collects upstream and downstream lineage information for views . + + Args: + view (str): name of the view + + """ + + url = self.config.get_sql_alchemy_url() + engine = create_engine(url, **self.config.options) + + get_refrence_table = sql.text( + dedent( + """ select reference_table_name from v_catalog.view_tables where table_name = '%(view)s' """ + % {"view": view} + ) + ) + + refrence_table = "" + for data in engine.execute(get_refrence_table): + # refrence_table.append(data) + refrence_table = data["reference_table_name"] + + view_upstream_lineage_query = sql.text( dedent( """ - SELECT VIEW_DEFINITION - FROM V_CATALOG.VIEWS - WHERE table_name='%(view_name)s' AND %(schema_condition)s + select reference_table_name ,reference_table_schema from v_catalog.view_tables where table_name = '%(view)s' """ + % {"view": view} + ) + ) + + view_downstream_query = sql.text( + dedent( """ - % {"view_name": view_name, "schema_condition": schema_condition} + select table_name ,table_schema from v_catalog.view_tables where reference_table_name = '%(view)s' + """ + % {"view": refrence_table} ) ) - ) + num_edges: int = 0 + + try: + self.view_lineage_map = defaultdict(list) + for db_row_key in engine.execute(view_downstream_query): + + downstream = f"{db_row_key['table_schema']}.{db_row_key['table_name']}" + + for db_row_value in engine.execute(view_upstream_lineage_query): - return view_def + upstream = f"{db_row_value['reference_table_schema']}.{db_row_value['reference_table_name']}" + view_upstream: str = upstream + view_name: str = downstream + self.view_lineage_map[view_name].append( + # (, , ) + (view_upstream, "[]", "[]") + ) -def get_columns(self, connection, table_name, schema=None, **kw): - if schema is not None: - schema_condition = "lower(table_schema) = '%(schema)s'" % { - "schema": schema.lower() - } - else: - schema_condition = "1" + num_edges += 1 - sql_pk_column = sql.text( - dedent( - """ - SELECT column_name - FROM v_catalog.primary_keys - WHERE table_name = '%(table_name)s' - AND constraint_type = 'p' - AND %(schema_condition)s - """ - % {"table_name": table_name, "schema_condition": schema_condition} + except Exception as e: + self.warn( + logger, + "view_upstream_lineage", + "Extracting the upstream & Downstream view lineage from vertica failed." + + f"Please check your permissions. Continuing...\nError was {e}.", + ) + + logger.info( + f"A total of {num_edges} View upstream edges found found for {view}" ) - ) - primary_key_columns = tuple(row[0] for row in connection.execute(sql_pk_column)) - - sql_get_column = sql.text( - dedent( - """ - SELECT column_name, data_type, column_default, is_nullable - FROM v_catalog.columns - WHERE table_name = '%(table_name)s' - AND %(schema_condition)s - UNION ALL - SELECT column_name, data_type, '' as column_default, true as is_nullable - FROM v_catalog.view_columns - WHERE lower(table_name) = '%(table_name)s' - AND %(schema_condition)s - """ - % {"table_name": table_name, "schema_condition": schema_condition} + + def _get_upstream_lineage_info_projection( + self, dataset_urn: str, projection: str + ) -> Optional[_Aspect]: + + dataset_key = dataset_urn_to_key(dataset_urn) + if dataset_key is None: + logger.warning(f"Invalid dataset urn {dataset_urn}. Could not get key!") + return None + + self._populate_projection_lineage(projection) + dataset_name = dataset_key.name + lineage = self.projection_lineage_map[dataset_name] # type: ignore + + if not (lineage): + logger.debug(f"No lineage found for {dataset_name}") + return None + upstream_tables: List[UpstreamClass] = [] + + for lineage_entry in lineage: + # Update the projection-lineage + upstream_table_name = lineage_entry[0] + + upstream_table = UpstreamClass( + dataset=make_dataset_urn_with_platform_instance( + self.platform, + upstream_table_name, + self.config.platform_instance, + self.config.env, + ), + type=DatasetLineageTypeClass.TRANSFORMED, + ) + upstream_tables.append(upstream_table) + + if upstream_tables: + + logger.debug( + f"lineage of Projection '{dataset_name}': {[u.dataset for u in upstream_tables]}" + ) + return UpstreamLineage(upstreams=upstream_tables) + return None + + def _populate_projection_lineage(self, projection: str) -> None: + """ + Collects upstream and downstream lineage information for views . + + Args: + projection (str): name of the projection + + + """ + url = self.config.get_sql_alchemy_url() + logger.debug(f"sql_alchemy_url={url}") + engine = create_engine(url, **self.config.options) + + view_upstream_lineage_query = sql.text( + dedent( + """ select basename , schemaname from vs_projections where name ='%(projection)s' + """ + % {"projection": projection} + ) ) - ) - columns = [] - for row in list(connection.execute(sql_get_column)): - name = row.column_name - primary_key = name in primary_key_columns - type = row.data_type.lower() - default = row.column_default - nullable = row.is_nullable - - column_info = self._get_column_info(name, type, default, nullable, schema) - column_info.update({"primary_key": primary_key}) - columns.append(column_info) - - return columns - - -def _get_column_info( # noqa: C901 - self, name, data_type, default, is_nullable, schema=None -): - attype: str = re.sub(r"\(.*\)", "", data_type) - - charlen = re.search(r"\(([\d,]+)\)", data_type) - if charlen: - charlen = charlen.group(1) # type: ignore - args = re.search(r"\((.*)\)", data_type) - if args and args.group(1): - args = tuple(re.split(r"\s*,\s*", args.group(1))) # type: ignore - else: - args = () # type: ignore - kwargs: Dict[str, Any] = {} - - if attype == "numeric": - if charlen: - prec, scale = charlen.split(",") # type: ignore - args = (int(prec), int(scale)) # type: ignore - else: - args = () # type: ignore - elif attype == "integer": - args = () # type: ignore - elif attype in ("timestamptz", "timetz"): - kwargs["timezone"] = True - if charlen: - kwargs["precision"] = int(charlen) # type: ignore - args = () # type: ignore - elif attype in ("timestamp", "time"): - kwargs["timezone"] = False - if charlen: - kwargs["precision"] = int(charlen) # type: ignore - args = () # type: ignore - elif attype.startswith("interval"): - field_match = re.match(r"interval (.+)", attype, re.I) - if charlen: - kwargs["precision"] = int(charlen) # type: ignore - if field_match: - kwargs["fields"] = field_match.group(1) # type: ignore - attype = "interval" - args = () # type: ignore - elif attype == "date": - args = () # type: ignore - elif charlen: - args = (int(charlen),) # type: ignore - - while True: - if attype.upper() in self.ischema_names: - coltype = self.ischema_names[attype.upper()] - break - else: - coltype = None - break - - self.ischema_names["UUID"] = UUID - self.ischema_names["TIMESTAMP"] = TIMESTAMP_WITH_PRECISION - self.ischema_names["TIMESTAMPTZ"] = TIMESTAMP_WITH_TIMEZONE - self.ischema_names["TIMETZ"] = TIME_WITH_TIMEZONE - - if coltype: - coltype = coltype(*args, **kwargs) - else: - util.warn("Did not recognize type '%s' of column '%s'" % (attype, name)) - coltype = sqltypes.NULLTYPE - # adjust the default value - autoincrement = False - if default is not None: - match = re.search(r"""(nextval\(')([^']+)('.*$)""", default) - if match is not None: - if issubclass(coltype._type_affinity, sqltypes.Integer): - autoincrement = True - # the default is related to a Sequence - sch = schema - if "." not in match.group(2) and sch is not None: - # unconditionally quote the schema name. this could - # later be enhanced to obey quoting rules / - # "quote schema" - default = ( - match.group(1) - + ('"%s"' % sch) - + "." - + match.group(2) - + match.group(3) + num_edges: int = 0 + + try: + self.projection_lineage_map = defaultdict(list) + for db_row_key in engine.execute(view_upstream_lineage_query): + basename = db_row_key["basename"] + upstream = f"{db_row_key['schemaname']}.{db_row_key['basename']}" + + view_downstream_query = sql.text( + dedent( + """ + select name,schemaname + from vs_projections + where basename='%(basename)s' + """ + % {"basename": basename} + ) ) + for db_row_value in engine.execute(view_downstream_query): + downstream = f"{db_row_value['schemaname']}.{db_row_value['name']}" + projection_upstream: str = upstream + projection_name: str = downstream + self.projection_lineage_map[projection_name].append( + # (, , ) + (projection_upstream, "[]", "[]") + ) + num_edges += 1 - column_info = dict( - name=name, - type=coltype, - nullable=is_nullable, - default=default, - autoincrement=autoincrement, - ) - return column_info + except Exception as error: + logger.warning( + "Extracting the upstream & downstream Projection lineage from Vertica failed %s", + error, + ) + logger.info( + f"A total of {num_edges} Projection lineage edges found for {projection}." + ) -VerticaDialect.get_view_definition = get_view_definition -# VerticaDialect.get_columns = get_columns -VerticaDialect._get_column_info = _get_column_info + def _get_owner_information(self, table: str, label: str) -> Optional[str]: + url = self.config.get_sql_alchemy_url() + engine = create_engine(url, **self.config.options) + if label == "table": + get_owner_query = sql.text( + dedent( + """ + SELECT owner_name + FROM v_catalog.tables + WHERE table_name = '%(table)s' + """ + % {"table": table} + ) + ) + for each in engine.execute(get_owner_query): + return each["owner_name"] + elif label == "view": + get_owner_query = sql.text( + dedent( + """ + SELECT owner_name + FROM v_catalog.views + WHERE table_name = '%(view)s' + """ + % {"view": table} + ) + ) -class VerticaConfig(BasicSQLAlchemyConfig): - # defaults - scheme: str = pydantic.Field(default="vertica+vertica_python") + for each in engine.execute(get_owner_query): + return each["owner_name"] - @validator("host_port") - def clean_host_port(cls, v): - return config_clean.remove_protocol(v) + elif label == "projection": + get_owner_query = sql.text( + dedent( + """ + SELECT owner_name + FROM v_catalog.projections + WHERE projection_name = '%(projection)s' + """ + % {"projection": table} + ) + ) + for each in engine.execute(get_owner_query): + return each["owner_name"] -@platform_name("Vertica") -@config_class(VerticaConfig) -@support_status(SupportStatus.TESTING) -@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") -@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") -class VerticaSource(SQLAlchemySource): - def __init__(self, config: VerticaConfig, ctx: PipelineContext) -> None: - super().__init__(config, ctx, "vertica") + return None - @classmethod - def create(cls, config_dict: Dict, ctx: PipelineContext) -> "VerticaSource": - config = VerticaConfig.parse_obj(config_dict) - return cls(config, ctx) + def close(self): + self.prepare_for_commit() diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index 42620d24a974a..3c9f3011204d4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -15,7 +15,6 @@ PlatformKey, UnitySchemaKey, add_dataset_to_container, - add_domain_to_entity_wu, gen_containers, ) from datahub.ingestion.api.common import PipelineContext @@ -45,16 +44,15 @@ from datahub.ingestion.source.unity.config import UnityCatalogSourceConfig from datahub.ingestion.source.unity.proxy import Catalog, Metastore, Schema from datahub.ingestion.source.unity.report import UnityCatalogReport -from datahub.metadata.com.linkedin.pegasus2avro.common import Status from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( FineGrainedLineage, FineGrainedLineageUpstreamType, ViewProperties, ) from datahub.metadata.schema_classes import ( - ChangeTypeClass, DatasetLineageTypeClass, DatasetPropertiesClass, + DomainsClass, MySqlDDLClass, SchemaFieldClass, SchemaMetadataClass, @@ -67,6 +65,7 @@ from datahub.utilities.source_helpers import ( auto_stale_entity_removal, auto_status_aspect, + auto_workunit_reporter, ) logger: logging.Logger = logging.getLogger(__name__) @@ -95,24 +94,11 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource): - tables and column lineage """ - def get_platform_instance_id(self) -> str: - return self.config.platform_instance or self.platform - config: UnityCatalogSourceConfig unity_catalog_api_proxy: proxy.UnityCatalogApiProxy platform: str = "databricks" platform_instance_name: str - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - # emit metadata work unit to DataHub GMS - yield from self.process_metastores() - - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_stale_entity_removal( - self.stale_entity_removal_handler, - auto_status_aspect(self.get_workunits_internal()), - ) - def get_report(self) -> SourceReport: return self.report @@ -171,10 +157,25 @@ def create(cls, config_dict, ctx): config = UnityCatalogSourceConfig.parse_obj(config_dict) return cls(ctx=ctx, config=config) + def get_platform_instance_id(self) -> str: + return self.config.platform_instance or self.platform + + def get_workunits(self) -> Iterable[MetadataWorkUnit]: + return auto_stale_entity_removal( + self.stale_entity_removal_handler, + auto_workunit_reporter( + self.report, + auto_status_aspect(self.get_workunits_internal()), + ), + ) + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + yield from self.process_metastores() + def process_metastores(self) -> Iterable[MetadataWorkUnit]: for metastore in self.unity_catalog_api_proxy.metastores(): if not self.config.metastore_id_pattern.allowed(metastore.metastore_id): - self.report.filtered.append(f"{metastore.metastore_id}.*.*.*") + self.report.report_dropped(f"{metastore.metastore_id}.*.*.*") continue logger.info( f"Started to process metastore: {metastore.metastore_id} ({metastore.name})" @@ -191,7 +192,7 @@ def process_catalogs( ) -> Iterable[MetadataWorkUnit]: for catalog in self.unity_catalog_api_proxy.catalogs(metastore=metastore): if not self.config.catalog_pattern.allowed(catalog.name): - self.report.filtered.append(f"{catalog.name}.*.*") + self.report.report_dropped(f"{catalog.name}.*.*") continue yield from self.gen_catalog_containers(catalog) self.report.increment_scanned_catalog(1) @@ -200,7 +201,7 @@ def process_catalogs( def process_schemas(self, catalog: proxy.Catalog) -> Iterable[MetadataWorkUnit]: for schema in self.unity_catalog_api_proxy.schemas(catalog=catalog): if not self.config.schema_pattern.allowed(schema.name): - self.report.filtered.append(f"{catalog.name}.{schema.name}.*") + self.report.report_dropped(f"{catalog.name}.{schema.name}.*") continue yield from self.gen_schema_containers(schema) @@ -213,7 +214,7 @@ def process_tables(self, schema: proxy.Schema) -> Iterable[MetadataWorkUnit]: if not self.config.table_pattern.allowed( f"{table.schema.catalog.name}.{table.schema.name}.{table.name}" ): - self.report.filtered.append( + self.report.report_dropped( f"{schema.catalog.name}.{schema.name}.{table.name}" ) continue @@ -224,37 +225,43 @@ def process_tables(self, schema: proxy.Schema) -> Iterable[MetadataWorkUnit]: name=table.id, ) yield from self.add_table_to_dataset_container(dataset_urn, schema) - yield self._create_table_property_aspect_mcp(table) + + table_props = self._create_table_property_aspect(table) + + view_props = None if table.view_definition: - yield self._create_view_property_aspect(table) - yield self._create_table_sub_type_aspect_mcp(table) - yield self._create_schema_metadata_aspect_mcp(table) - status = Status(removed=False) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=status, - ) + view_props = self._create_view_property_aspect(table) - wu = MetadataWorkUnit(id=f"status-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - yield wu + sub_type = self._create_table_sub_type_aspect(table) + schema_metadata = self._create_schema_metadata_aspect(table) - yield from self._get_domain_wu( + domain = self._get_domain_aspect( dataset_name=str( f"{table.schema.catalog.name}.{table.schema.name}.{table.name}" - ), - entity_urn=dataset_urn, - entity_type="dataset", + ) ) if self.config.include_column_lineage: self.unity_catalog_api_proxy.get_column_lineage(table) - yield from self._generate_column_lineage_mcp(dataset_urn, table) + lineage = self._generate_column_lineage_aspect(dataset_urn, table) else: self.unity_catalog_api_proxy.table_lineage(table) - yield from self._generate_lineage_mcp(dataset_urn, table) + lineage = self._generate_lineage_aspect(dataset_urn, table) + + yield from [ + mcp.as_workunit() + for mcp in MetadataChangeProposalWrapper.construct_many( + entityUrn=dataset_urn, + aspects=[ + table_props, + view_props, + sub_type, + schema_metadata, + domain, + lineage, + ], + ) + ] self.report.report_entity_scanned( f"{table.schema.catalog.name}.{table.schema.name}.{table.name}", @@ -263,9 +270,9 @@ def process_tables(self, schema: proxy.Schema) -> Iterable[MetadataWorkUnit]: self.report.increment_scanned_table(1) - def _generate_column_lineage_mcp( + def _generate_column_lineage_aspect( self, dataset_urn: str, table: proxy.Table - ) -> Iterable[MetadataWorkUnit]: + ) -> Optional[UpstreamLineageClass]: upstreams: List[UpstreamClass] = [] finegrained_lineages: List[FineGrainedLineage] = [] for upstream in sorted(table.upstreams.keys()): @@ -294,22 +301,15 @@ def _generate_column_lineage_mcp( upstreams.append(upstream_table) if upstreams: - upstream_lineage = UpstreamLineageClass( + return UpstreamLineageClass( upstreams=upstreams, fineGrainedLineages=finegrained_lineages ) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=upstream_lineage, - ) - wu = MetadataWorkUnit(id=f"upstream-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - yield wu + else: + return None - def _generate_lineage_mcp( + def _generate_lineage_aspect( self, dataset_urn: str, table: proxy.Table - ) -> Iterable[MetadataWorkUnit]: + ) -> Optional[UpstreamLineageClass]: upstreams: List[UpstreamClass] = [] for upstream in sorted(table.upstreams.keys()): upstream_urn = make_dataset_urn_with_platform_instance( @@ -325,39 +325,21 @@ def _generate_lineage_mcp( upstreams.append(upstream_table) if upstreams: - upstream_lineage = UpstreamLineageClass(upstreams=upstreams) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=upstream_lineage, - ) - wu = MetadataWorkUnit(id=f"upstream-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - yield wu - - def _get_domain_wu( - self, - dataset_name: str, - entity_urn: str, - entity_type: str, - ) -> Iterable[MetadataWorkUnit]: + return UpstreamLineageClass(upstreams=upstreams) + else: + return None + + def _get_domain_aspect(self, dataset_name: str) -> Optional[DomainsClass]: domain_urn = self._gen_domain_urn(dataset_name) - if domain_urn: - wus = add_domain_to_entity_wu( - entity_type=entity_type, - entity_urn=entity_urn, - domain_urn=domain_urn, - ) - for wu in wus: - self.report.report_workunit(wu) - yield wu + if not domain_urn: + return None + return DomainsClass(domains=[domain_urn]) def gen_schema_containers(self, schema: Schema) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(f"{schema.catalog.name}.{schema.name}") schema_container_key = self.gen_schema_key(schema) - container_workunits = gen_containers( + yield from gen_containers( container_key=schema_container_key, name=schema.name, sub_types=["Schema"], @@ -366,10 +348,6 @@ def gen_schema_containers(self, schema: Schema) -> Iterable[MetadataWorkUnit]: description=schema.comment, ) - for wu in container_workunits: - self.report.report_workunit(wu) - yield wu - def gen_metastore_containers( self, metastore: Metastore ) -> Iterable[MetadataWorkUnit]: @@ -377,7 +355,7 @@ def gen_metastore_containers( metastore_container_key = self.gen_metastore_key(metastore) - container_workunits = gen_containers( + yield from gen_containers( container_key=metastore_container_key, name=metastore.name, sub_types=["Metastore"], @@ -385,10 +363,6 @@ def gen_metastore_containers( description=metastore.comment, ) - for wu in container_workunits: - self.report.report_workunit(wu) - yield wu - def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(catalog.name) @@ -396,7 +370,7 @@ def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit] catalog_container_key = self.gen_catalog_key(catalog) - container_workunits = gen_containers( + yield from gen_containers( container_key=catalog_container_key, name=catalog.name, sub_types=["Catalog"], @@ -405,10 +379,6 @@ def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit] description=catalog.comment, ) - for wu in container_workunits: - self.report.report_workunit(wu) - yield wu - def gen_schema_key(self, schema: Schema) -> PlatformKey: return UnitySchemaKey( unity_schema=schema.name, @@ -418,14 +388,14 @@ def gen_schema_key(self, schema: Schema) -> PlatformKey: metastore=schema.catalog.metastore.name, ) - def gen_metastore_key(self, metastore: Metastore) -> PlatformKey: + def gen_metastore_key(self, metastore: Metastore) -> MetastoreKey: return MetastoreKey( metastore=metastore.name, platform=self.platform, instance=self.config.platform_instance, ) - def gen_catalog_key(self, catalog: Catalog) -> PlatformKey: + def gen_catalog_key(self, catalog: Catalog) -> CatalogKey: return CatalogKey( catalog=catalog.name, metastore=catalog.metastore.name, @@ -448,20 +418,14 @@ def add_table_to_dataset_container( self, dataset_urn: str, schema: Schema ) -> Iterable[MetadataWorkUnit]: schema_container_key = self.gen_schema_key(schema) - container_workunits = add_dataset_to_container( + yield from add_dataset_to_container( container_key=schema_container_key, dataset_urn=dataset_urn, ) - for wu in container_workunits: - self.report.report_workunit(wu) - yield wu - def _create_table_property_aspect_mcp(self, table: proxy.Table) -> MetadataWorkUnit: - dataset_urn: str = make_dataset_urn_with_platform_instance( - platform=self.platform, - platform_instance=self.platform_instance_name, - name=table.id, - ) + def _create_table_property_aspect( + self, table: proxy.Table + ) -> DatasetPropertiesClass: custom_properties: dict = {} if table.storage_location is not None: custom_properties["storage_location"] = table.storage_location @@ -480,93 +444,37 @@ def _create_table_property_aspect_mcp(self, table: proxy.Table) -> MetadataWorkU custom_properties["updated_by"] = table.updated_by custom_properties["updated_at"] = str(table.updated_at) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=DatasetPropertiesClass( - name=table.name, - description=table.comment, - customProperties=custom_properties, - ), + return DatasetPropertiesClass( + name=table.name, + description=table.comment, + customProperties=custom_properties, ) - wu = MetadataWorkUnit(id=f"datasetProperties-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - - return wu - - def _create_table_sub_type_aspect_mcp(self, table: proxy.Table) -> MetadataWorkUnit: - dataset_urn: str = make_dataset_urn_with_platform_instance( - platform=self.platform, - platform_instance=self.platform_instance_name, - name=table.id, - ) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=SubTypesClass( - typeNames=["View" if table.table_type.lower() == "view" else "Table"] - ), + def _create_table_sub_type_aspect(self, table: proxy.Table) -> SubTypesClass: + return SubTypesClass( + typeNames=["View" if table.table_type.lower() == "view" else "Table"] ) - wu = MetadataWorkUnit(id=f"subType-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - - return wu - - def _create_view_property_aspect(self, table: proxy.Table) -> MetadataWorkUnit: - dataset_urn: str = make_dataset_urn_with_platform_instance( - platform=self.platform, - platform_instance=self.platform_instance_name, - name=table.id, - ) + def _create_view_property_aspect(self, table: proxy.Table) -> ViewProperties: assert table.view_definition - view_properties_aspect = ViewProperties( + return ViewProperties( materialized=False, viewLanguage="SQL", viewLogic=table.view_definition ) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=view_properties_aspect, - ) - wu = MetadataWorkUnit(id=f"view_properties-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - - return wu - def _create_schema_metadata_aspect_mcp( - self, table: proxy.Table - ) -> MetadataWorkUnit: + def _create_schema_metadata_aspect(self, table: proxy.Table) -> SchemaMetadataClass: schema_fields: List[SchemaFieldClass] = [] for column in table.columns: schema_fields.extend(self._create_schema_field(column)) - dataset_urn: str = make_dataset_urn_with_platform_instance( - platform=self.platform, - platform_instance=self.platform_instance_name, - name=table.id, + return SchemaMetadataClass( + schemaName=table.id, + platform=make_data_platform_urn(self.platform), + fields=schema_fields, + hash="", + version=0, + platformSchema=MySqlDDLClass(tableSchema=""), ) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspect=SchemaMetadataClass( - schemaName=table.id, - platform=make_data_platform_urn(self.platform), - fields=schema_fields, - hash="", - version=0, - platformSchema=MySqlDDLClass(tableSchema=""), - ), - ) - wu = MetadataWorkUnit(id=f"schema_metaclass-{dataset_urn}", mcp=mcp) - self.report.report_workunit(wu) - - return wu @staticmethod def _create_schema_field(column: proxy.Column) -> List[SchemaFieldClass]: diff --git a/metadata-ingestion/src/datahub/utilities/source_helpers.py b/metadata-ingestion/src/datahub/utilities/source_helpers.py index 2b771aa2a9f75..4f7d9ae309d4a 100644 --- a/metadata-ingestion/src/datahub/utilities/source_helpers.py +++ b/metadata-ingestion/src/datahub/utilities/source_helpers.py @@ -1,6 +1,7 @@ from typing import Callable, Iterable, Optional, Set, Union from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.source import SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, @@ -87,3 +88,16 @@ def auto_stale_entity_removal( # Clean up stale entities. yield from stale_entity_removal_handler.gen_removed_entity_workunits() + + +def auto_workunit_reporter( + report: SourceReport, + stream: Iterable[MetadataWorkUnit], +) -> Iterable[MetadataWorkUnit]: + """ + Calls report.report_workunit() on each workunit. + """ + + for wu in stream: + report.report_workunit(wu) + yield wu diff --git a/metadata-ingestion/tests/integration/git/test_git_clone.py b/metadata-ingestion/tests/integration/git/test_git_clone.py index e9ccc71252887..3cd67af666fd3 100644 --- a/metadata-ingestion/tests/integration/git/test_git_clone.py +++ b/metadata-ingestion/tests/integration/git/test_git_clone.py @@ -1,31 +1,50 @@ import os +import pytest from pydantic import SecretStr +from datahub.configuration.github import GitHubInfo from datahub.ingestion.source.git.git_import import GitClone +LOOKML_TEST_SSH_KEY = os.environ.get("DATAHUB_LOOKML_GIT_TEST_SSH_KEY") -def test_git_clone(pytestconfig, tmp_path): + +@pytest.mark.skipif( + LOOKML_TEST_SSH_KEY is None, + reason="DATAHUB_LOOKML_GIT_TEST_SSH_KEY env variable is not configured", +) +def test_git_clone(tmp_path): git_clone = GitClone(str(tmp_path)) - secret_env_variable = "DATAHUB_LOOKML_GIT_TEST_SSH_KEY" - if os.environ.get(secret_env_variable) is not None: - secret_key = SecretStr(os.environ.get(secret_env_variable)) # type: ignore - checkout_dir = git_clone.clone( - ssh_key=secret_key, - repo_url="git@github.com:acryldata/long-tail-companions-looker", - ) - assert os.path.exists(checkout_dir) - assert set(os.listdir(checkout_dir)) == set( - [ - ".datahub", - "models", - "README.md", - ".github", - ".git", - "views", - ] - ) - else: - print( - "Skipping test as env variable DATAHUB_LOOKML_GIT_TEST_SSH_KEY is not configured" - ) + secret_key = SecretStr(LOOKML_TEST_SSH_KEY) if LOOKML_TEST_SSH_KEY else None + + checkout_dir = git_clone.clone( + ssh_key=secret_key, + repo_url="git@github.com:acryldata/long-tail-companions-looker", + branch="d380a2b777ec6f4653626f39c68dba85893faa74", + ) + assert os.path.exists(checkout_dir) + assert set(os.listdir(checkout_dir)) == set( + [ + ".datahub", + "models", + "README.md", + ".github", + ".git", + "views", + "manifest_lock.lkml", + "manifest.lkml", + ] + ) + + +def test_github_branch(): + config = GitHubInfo( + repo="owner/repo", + ) + assert config.branch_for_clone is None + + config = GitHubInfo( + repo="owner/repo", + branch="main", + ) + assert config.branch_for_clone == "main" diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 285c1657ba7e1..e3cb3639d9a93 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -352,5 +352,60 @@ def default_query_results(query): # noqa: C901 ]: return [] + elif ( + query + == snowflake_query.SnowflakeQuery.get_all_tags_in_database_without_propagation( + "TEST_DB" + ) + ): + return [ + *[ + { + "TAG_DATABASE": "TEST_DB", + "TAG_SCHEMA": "TEST_SCHEMA", + "TAG_NAME": f"my_tag_{ix}", + "TAG_VALUE": f"my_value_{ix}", + "OBJECT_DATABASE": "TEST_DB", + "OBJECT_SCHEMA": "TEST_SCHEMA", + "OBJECT_NAME": "VIEW_2", + "COLUMN_NAME": None, + "DOMAIN": "TABLE", + } + for ix in range(3) + ], + { + "TAG_DATABASE": "TEST_DB", + "TAG_SCHEMA": "TEST_SCHEMA", + "TAG_NAME": "security", + "TAG_VALUE": "pii", + "OBJECT_DATABASE": "TEST_DB", + "OBJECT_SCHEMA": "TEST_SCHEMA", + "OBJECT_NAME": "VIEW_1", + "COLUMN_NAME": "COL_1", + "DOMAIN": "COLUMN", + }, + { + "TAG_DATABASE": "OTHER_DB", + "TAG_SCHEMA": "OTHER_SCHEMA", + "TAG_NAME": "my_other_tag", + "TAG_VALUE": "other", + "OBJECT_DATABASE": "TEST_DB", + "OBJECT_SCHEMA": None, + "OBJECT_NAME": "TEST_SCHEMA", + "COLUMN_NAME": None, + "DOMAIN": "SCHEMA", + }, + { + "TAG_DATABASE": "OTHER_DB", + "TAG_SCHEMA": "OTHER_SCHEMA", + "TAG_NAME": "my_other_tag", + "TAG_VALUE": "other", + "OBJECT_DATABASE": None, + "OBJECT_SCHEMA": None, + "OBJECT_NAME": "TEST_DB", + "COLUMN_NAME": None, + "DOMAIN": "DATABASE", + }, + ] # Unreachable code raise Exception(f"Unknown query {query}") diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 7988a135f3476..c4bbe24980d35 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -1,1388 +1,4276 @@ [ { - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"snowflake\", \"instance\": \"PROD\", \"database\": \"test_db\"}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/\", \"name\": \"TEST_DB\", \"description\": \"Comment for TEST_DB\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:snowflake\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"snowflake\", \"instance\": \"PROD\", \"database\": \"test_db\", \"schema\": \"test_schema\"}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/\", \"name\": \"TEST_SCHEMA\", \"description\": \"comment for TEST_DB.TEST_SCHEMA\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:snowflake\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:5e359958be02ce647cd9ac196dbd4585\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_1\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/\", \"name\": \"TABLE_1\", \"qualifiedName\": \"test_db.test_schema.table_1\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_2\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/\", \"name\": \"TABLE_2\", \"qualifiedName\": \"test_db.test_schema.table_2\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_3\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/\", \"name\": \"TABLE_3\", \"qualifiedName\": \"test_db.test_schema.table_3\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_4\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/\", \"name\": \"TABLE_4\", \"qualifiedName\": \"test_db.test_schema.table_4\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_5\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/\", \"name\": \"TABLE_5\", \"qualifiedName\": \"test_db.test_schema.table_5\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_6\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/\", \"name\": \"TABLE_6\", \"qualifiedName\": \"test_db.test_schema.table_6\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_7\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/\", \"name\": \"TABLE_7\", \"qualifiedName\": \"test_db.test_schema.table_7\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_8\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/\", \"name\": \"TABLE_8\", \"qualifiedName\": \"test_db.test_schema.table_8\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_9\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/\", \"name\": \"TABLE_9\", \"qualifiedName\": \"test_db.test_schema.table_9\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.table_10\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Age\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:Email_Address\"}], \"auditStamp\": {\"time\": 1654621200000, \"actor\": \"urn:li:corpuser:datahub\"}}, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/\", \"name\": \"TABLE_10\", \"qualifiedName\": \"test_db.test_schema.table_10\", \"description\": \"Comment for Table\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.view_1\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/\", \"name\": \"VIEW_1\", \"qualifiedName\": \"test_db.test_schema.view_1\", \"description\": \"Comment for View\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "value": "{\"schemaName\": \"test_db.test_schema.view_2\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"NUMBER(38,0)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR(255)\", \"recursive\": false, \"isPartOfKey\": false}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/\", \"name\": \"VIEW_2\", \"qualifiedName\": \"test_db.test_schema.view_2\", \"description\": \"Comment for View\", \"created\": {\"time\": 1623110400000}, \"lastModified\": {\"time\": 1623110400000}, \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10000, \"columnCount\": 10, \"sizeInBytes\": 1024}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}], \"fineGrainedLineages\": [{\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)\"], \"confidenceScore\": 1.0}, {\"upstreamType\": \"FIELD_SET\", \"upstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)\"], \"downstreamType\": \"FIELD\", \"downstreams\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)\"], \"confidenceScore\": 1.0}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "snowflake", + "instance": "PROD", + "database": "test_db" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/", + "name": "TEST_DB", + "description": "Comment for TEST_DB", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 } } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { "json": { "platform": "urn:li:dataPlatform:snowflake" } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["Database"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { "tag": "urn:li:tag:other_db.other_schema.my_other_tag:other" } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "OTHER_DB.OTHER_SCHEMA.my_other_tag:other", + "description": "Represents the Snowflake tag `OTHER_DB.OTHER_SCHEMA.my_other_tag` with value `other`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "snowflake", + "instance": "PROD", + "database": "test_db", + "schema": "test_schema" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/", + "name": "TEST_SCHEMA", + "description": "comment for TEST_DB.TEST_SCHEMA", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 } + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { "json": { "platform": "urn:li:dataPlatform:snowflake" } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["Schema"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { "tag": "urn:li:tag:other_db.other_schema.my_other_tag:other" } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/", + "name": "TABLE_1", + "qualifiedName": "test_db.test_schema.table_1", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_2", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/", + "name": "TABLE_2", + "qualifiedName": "test_db.test_schema.table_2", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_3", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/", + "name": "TABLE_3", + "qualifiedName": "test_db.test_schema.table_3", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_4", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/", + "name": "TABLE_4", + "qualifiedName": "test_db.test_schema.table_4", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_5", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/", + "name": "TABLE_5", + "qualifiedName": "test_db.test_schema.table_5", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_6", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/", + "name": "TABLE_6", + "qualifiedName": "test_db.test_schema.table_6", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_7", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/", + "name": "TABLE_7", + "qualifiedName": "test_db.test_schema.table_7", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_8", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/", + "name": "TABLE_8", + "qualifiedName": "test_db.test_schema.table_8", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_9", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/", + "name": "TABLE_9", + "qualifiedName": "test_db.test_schema.table_9", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_10", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Age" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [{ "urn": "urn:li:glossaryTerm:Email_Address" }], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/", + "name": "TABLE_10", + "qualifiedName": "test_db.test_schema.table_10", + "description": "Comment for Table", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["table"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "TEST_DB.TEST_SCHEMA.security:pii", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.security` with value `pii`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.view_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "globalTags": { + "tags": [{ "tag": "urn:li:tag:test_db.test_schema.security:pii" }] + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/", + "name": "VIEW_1", + "qualifiedName": "test_db.test_schema.view_1", + "description": "Comment for View", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["view"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "TEST_DB.TEST_SCHEMA.my_tag_0:my_value_0", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_0` with value `my_value_0`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "TEST_DB.TEST_SCHEMA.my_tag_1:my_value_1", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_1` with value `my_value_1`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "TEST_DB.TEST_SCHEMA.my_tag_2:my_value_2", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_2` with value `my_value_2`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { "json": { "removed": false } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.view_2", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { "tableSchema": "" } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.NumberType": {} } }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { "type": { "com.linkedin.schema.StringType": {} } }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/", + "name": "VIEW_2", + "qualifiedName": "test_db.test_schema.view_2", + "description": "Comment for View", + "created": { "time": 1623110400000 }, + "lastModified": { "time": 1623110400000 }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { "json": { "typeNames": ["view"] } }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { "tag": "urn:li:tag:TEST_DB.TEST_SCHEMA.my_tag_0:my_value_0" }, + { "tag": "urn:li:tag:TEST_DB.TEST_SCHEMA.my_tag_1:my_value_1" }, + { "tag": "urn:li:tag:TEST_DB.TEST_SCHEMA.my_tag_2:my_value_2" } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown" }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } } -] \ No newline at end of file +] diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index ec9e342cbea56..702701246baa9 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -19,7 +19,10 @@ from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.run.pipeline_config import PipelineConfig, SourceConfig from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig -from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config +from datahub.ingestion.source.snowflake.snowflake_config import ( + SnowflakeV2Config, + TagOption, +) from tests.integration.snowflake.common import FROZEN_TIME, default_query_results from tests.test_helpers import mce_helpers @@ -109,6 +112,7 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): profile_table_size_limit=None, profile_table_level_only=True, ), + extract_tags=TagOption.without_lineage, ), ), sink=DynamicTypedConfig( diff --git a/metadata-ingestion/tests/integration/vertica/docker-compose.yml b/metadata-ingestion/tests/integration/vertica/docker-compose.yml index 6f1e26dd8a2c1..1b22d804b0a37 100644 --- a/metadata-ingestion/tests/integration/vertica/docker-compose.yml +++ b/metadata-ingestion/tests/integration/vertica/docker-compose.yml @@ -1,25 +1,22 @@ -version: '3.9' +version: "3.9" services: vertica: - image: vertica/vertica-ce:10.1.1-0 - container_name: vertica_ce - platform: linux/amd64 environment: - - APP_DB_USER=dbadmin - - APP_DB_PASSWORD=vertica - - VERTICA_MEMDEBUG=2 + APP_DB_USER: "dbadmin" + APP_DB_PASSWORD: "vertica" + container_name: vertica-ce + image: vertica/vertica-ce:12.0.2-0 ports: - - 5433:5433 - - 5444:5444 + - "5433:5433" + - "5444:5444" deploy: mode: global volumes: - type: volume source: vertica-data target: /data + volumes: vertica-data: -networks: - default: - external: - name: datahub_network + + diff --git a/metadata-ingestion/tests/integration/vertica/sqlalchemy.json b/metadata-ingestion/tests/integration/vertica/sqlalchemy.json deleted file mode 100644 index a5986c3eacd42..0000000000000 --- a/metadata-ingestion/tests/integration/vertica/sqlalchemy.json +++ /dev/null @@ -1,135 +0,0 @@ -[ -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"vertica\", \"instance\": \"PROD\", \"database\": \"vmart\"}, \"name\": \"vmart\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612598, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:vertica\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612599, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612599, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"vertica\", \"instance\": \"PROD\", \"database\": \"vmart\", \"schema\": \"public\"}, \"name\": \"public\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612825, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:vertica\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612827, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612828, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:343f520ad0fb3259b298736800bb1385\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409612828, - "runId": "sqlalchemy-2022_04_08-09_20_06", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/sqlalchemy_to_file.yml b/metadata-ingestion/tests/integration/vertica/sqlalchemy_to_file.yml deleted file mode 100644 index 07154b6f64745..0000000000000 --- a/metadata-ingestion/tests/integration/vertica/sqlalchemy_to_file.yml +++ /dev/null @@ -1,10 +0,0 @@ -source: - type: sqlalchemy - config: - platform: "vertica" - connect_uri: "vertica+vertica_python://dbadmin:vertica@vertica_ce:5433/VMart" - -sink: - type: file - config: - filename: "./sqlalchemy.json" diff --git a/metadata-ingestion/tests/integration/vertica/test_vertica.py b/metadata-ingestion/tests/integration/vertica/test_vertica.py index 09f49eff1237a..46d7ad987ef8c 100644 --- a/metadata-ingestion/tests/integration/vertica/test_vertica.py +++ b/metadata-ingestion/tests/integration/vertica/test_vertica.py @@ -3,31 +3,46 @@ from tests.test_helpers import mce_helpers from tests.test_helpers.click_helpers import run_datahub_cmd -from tests.test_helpers.docker_helpers import wait_for_port +from tests.test_helpers.docker_helpers import is_responsive, wait_for_port FROZEN_TIME = "2020-04-14 07:00:00" -@freeze_time(FROZEN_TIME) -@pytest.mark.integration -@pytest.mark.skip("This does not work yet and needs to be fixed.") -def test_vertica_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): - test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica" +@pytest.fixture(scope="module") +def test_resources_dir(pytestconfig): + return pytestconfig.rootpath / "tests/integration/vertica" + +@pytest.fixture(scope="module") +def vertica_runner(docker_compose_runner, pytestconfig, test_resources_dir): with docker_compose_runner( test_resources_dir / "docker-compose.yml", "vertica" ) as docker_services: - wait_for_port(docker_services, "vertica_ce", 5433, timeout=120) - - # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "vertica_to_file.yml").resolve() - run_datahub_cmd( - ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path + wait_for_port( + docker_services, + "vertica-ce", + 5433, + timeout=120, + checker=lambda: is_responsive("vertica-ce", 5433, hostname="vertica-ce"), ) + yield docker_services - # Verify the output. - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / "vertica.json", - golden_path=test_resources_dir / "vertica_golden.json", - ) + +# Test needs more work to be done , currently it is working fine. +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +@pytest.mark.skip("This does not work yet and needs to be fixed.") +def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path): + test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica" + # Run the metadata ingestion pipeline. + config_file = (test_resources_dir / "vertica_to_file.yml").resolve() + run_datahub_cmd( + ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path + ) + + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "vertica.json", + golden_path=test_resources_dir / "vertica_mces_with_db_golden.json", + ) diff --git a/metadata-ingestion/tests/integration/vertica/vertica.json b/metadata-ingestion/tests/integration/vertica/vertica.json deleted file mode 100644 index bd63acca6332f..0000000000000 --- a/metadata-ingestion/tests/integration/vertica/vertica.json +++ /dev/null @@ -1,135 +0,0 @@ -[ -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"vertica\", \"instance\": \"PROD\", \"database\": \"vmart\"}, \"name\": \"vmart\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653057, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:vertica\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653060, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653064, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"vertica\", \"instance\": \"PROD\", \"database\": \"vmart\", \"schema\": \"public\"}, \"name\": \"public\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653169, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:vertica\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653177, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653183, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:343f520ad0fb3259b298736800bb1385\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1649409653184, - "runId": "vertica-2022_04_08-09_20_48", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json new file mode 100644 index 0000000000000..7c2de60372b67 --- /dev/null +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -0,0 +1,159 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "instance": "PROD", + "database": "vmart", + "cluster_type": "Enterprise", + "cluster_size": "87 GB", + "subcluster": " ", + "communal_storage_path": "" + }, + "name": "vmart" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "instance": "PROD", + "database": "vmart", + "schema": "public", + "projection_count": "0", + "udx_list": "", + "Udx_langauge": "" + }, + "name": "public" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml index 04c8c0bf35801..da1fcc2b5304f 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml +++ b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml @@ -1,8 +1,8 @@ source: type: "vertica" config: - host_port: vertica_ce:5433 - database: VMart + host_port: 127.0.0.1:5433 + database: Vmart username: dbadmin password: vertica diff --git a/metadata-ingestion/tests/unit/test_airflow.py b/metadata-ingestion/tests/unit/test_airflow.py index 823de9009a677..8cbe4752938bc 100644 --- a/metadata-ingestion/tests/unit/test_airflow.py +++ b/metadata-ingestion/tests/unit/test_airflow.py @@ -74,6 +74,7 @@ def test_airflow_provider_info(): assert get_provider_info() +@pytest.mark.filterwarnings("ignore:.*is deprecated.*") def test_dags_load_with_no_errors(pytestconfig: pytest.Config) -> None: airflow_examples_folder = ( pytestconfig.rootpath / "src/datahub_provider/example_dags" diff --git a/metadata-ingestion/tests/unit/test_kafka_sink.py b/metadata-ingestion/tests/unit/test_kafka_sink.py index 1e3d66fa50039..9f4062bf93bf8 100644 --- a/metadata-ingestion/tests/unit/test_kafka_sink.py +++ b/metadata-ingestion/tests/unit/test_kafka_sink.py @@ -39,10 +39,7 @@ def test_kafka_sink_mcp(self, mock_producer, mock_callback): from datahub.emitter.mcp import MetadataChangeProposalWrapper mcp = MetadataChangeProposalWrapper( - entityType="dataset", entityUrn="urn:li:dataset:(urn:li:dataPlatform:mysql,User.UserAccount,PROD)", - changeType=models.ChangeTypeClass.UPSERT, - aspectName="datasetProfile", aspect=models.DatasetProfileClass( rowCount=2000, columnCount=15, diff --git a/metadata-ingestion/tests/unit/test_plugin_system.py b/metadata-ingestion/tests/unit/test_plugin_system.py index 732cef123a90c..f5748ff5ea007 100644 --- a/metadata-ingestion/tests/unit/test_plugin_system.py +++ b/metadata-ingestion/tests/unit/test_plugin_system.py @@ -2,7 +2,7 @@ import pytest -from datahub.configuration.common import ConfigurationError +from datahub.configuration.common import ConfigurationError, ConfigurationWarning from datahub.ingestion.api.registry import PluginRegistry from datahub.ingestion.api.sink import Sink from datahub.ingestion.extractor.extractor_registry import extractor_registry @@ -97,9 +97,9 @@ class DummyClass: "console-alias", "console", lambda: warnings.warn( - UserWarning("console-alias is deprecated, use console instead") + ConfigurationWarning("console-alias is deprecated, use console instead") ), ) - with pytest.warns(UserWarning): + with pytest.warns(ConfigurationWarning): assert fake_registry.get("console-alias") == ConsoleSink assert "console-alias" not in fake_registry.summary(verbose=False) diff --git a/metadata-ingestion/tests/unit/test_rest_sink.py b/metadata-ingestion/tests/unit/test_rest_sink.py index fec2bf4784347..8390e8be0ed37 100644 --- a/metadata-ingestion/tests/unit/test_rest_sink.py +++ b/metadata-ingestion/tests/unit/test_rest_sink.py @@ -249,10 +249,7 @@ ), ( MetadataChangeProposalWrapper( - entityType="dataset", entityUrn="urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)", - changeType=models.ChangeTypeClass.UPSERT, - aspectName="ownership", aspect=models.OwnershipClass( owners=[ models.OwnerClass( diff --git a/metadata-ingestion/tests/unit/test_source.py b/metadata-ingestion/tests/unit/test_source.py index 41fc83caf3096..58c008049f2a6 100644 --- a/metadata-ingestion/tests/unit/test_source.py +++ b/metadata-ingestion/tests/unit/test_source.py @@ -4,7 +4,7 @@ from datahub.ingestion.api import workunit from datahub.ingestion.api.common import PipelineContext, WorkUnit from datahub.ingestion.api.source import Source, SourceReport -from datahub.metadata.schema_classes import ChangeTypeClass, StatusClass +from datahub.metadata.schema_classes import StatusClass from datahub.utilities.urns.dataset_urn import DatasetUrn @@ -14,8 +14,6 @@ def get_workunits(self) -> Iterable[WorkUnit]: workunit.MetadataWorkUnit( id="test-workunit", mcp=MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=str( DatasetUrn.create_from_ids( platform_id="elasticsearch", @@ -23,7 +21,6 @@ def get_workunits(self) -> Iterable[WorkUnit]: env="PROD", ) ), - aspectName="status", aspect=StatusClass(removed=False), ), ) diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index da5cd4343fbd4..493c3d086cea6 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -64,7 +64,6 @@ ) from datahub.metadata.schema_classes import ( BrowsePathsClass, - ChangeTypeClass, DatasetPropertiesClass, GlobalTagsClass, MetadataChangeEventClass, @@ -1301,8 +1300,6 @@ def test_mcp_multiple_transformers_replace(mock_time, tmp_path): Union[MetadataChangeEventClass, MetadataChangeProposalWrapper] ] = [ MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=str( DatasetUrn.create_from_ids( platform_id="elasticsearch", @@ -1310,7 +1307,6 @@ def test_mcp_multiple_transformers_replace(mock_time, tmp_path): env="PROD", ) ), - aspectName="globalTags", aspect=GlobalTagsClass(tags=[TagAssociationClass(tag="urn:li:tag:Test")]), ) for i in range(0, 10) @@ -1318,8 +1314,6 @@ def test_mcp_multiple_transformers_replace(mock_time, tmp_path): mcps.extend( [ MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=str( DatasetUrn.create_from_ids( platform_id="elasticsearch", @@ -1327,7 +1321,6 @@ def test_mcp_multiple_transformers_replace(mock_time, tmp_path): env="PROD", ) ), - aspectName="datasetProperties", aspect=DatasetPropertiesClass(description="test dataset"), ) for i in range(0, 10) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java new file mode 100644 index 0000000000000..ef03206996823 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java @@ -0,0 +1,12 @@ +package com.linkedin.metadata.config; + +import lombok.Data; + + +@Data +public class QueriesTabConfig { + /** + * The number of queries to show on the queries tab. + */ + public Integer queriesTabResultSize; +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java b/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java index 651d47dc3554c..6505fdf29da16 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java @@ -12,4 +12,9 @@ public class VisualConfiguration { * Asset related configurations */ public AssetsConfiguration assets; + + /** + * Queries tab related configurations + */ + public QueriesTabConfig queriesTab; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 666e67d016b78..71355aeebea22 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -6,7 +6,12 @@ import com.linkedin.metadata.models.annotation.SearchableAnnotation; import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; import javax.annotation.Nonnull; import com.linkedin.util.Pair; @@ -24,7 +29,6 @@ import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; -import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.TEXT_ANALYZER; public class SearchQueryBuilder { diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java index f3eb3a6408789..2d89abf7bae14 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java @@ -56,11 +56,9 @@ private static List> getPropertiesList(@Nullable final Strin @Nullable private static boolean isValueListValid(@Nullable final List entryList, final int valueListSize) { if (entryList == null) { - log.warn("Unable to get entry as entryList is null"); return false; } if (valueListSize != entryList.size()) { - log.warn("Unable to get entry for graph edge as values list and entry list have differing sizes"); return false; } return true; diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Embed.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Embed.pdl new file mode 100644 index 0000000000000..934ebcd50edd7 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Embed.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.common + +/** + * Information regarding rendering an embed for an asset. + */ +@Aspect = { + "name": "embed" +} +record Embed { + /** + * An embed URL to be rendered inside of an iframe. + */ + renderUrl: optional string +} \ No newline at end of file diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 71d6f8b20b6da..faad02b762b43 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -21,6 +21,7 @@ entities: - deprecation - testResults - siblings + - embed - name: dataHubPolicy doc: DataHub Policies represent access policies granted to users or groups on metadata operations like edit, view etc. category: internal @@ -59,6 +60,7 @@ entities: - deprecation - inputFields - chartUsageStatistics + - embed - name: dashboard keyAspect: dashboardKey aspects: @@ -68,6 +70,7 @@ entities: - dashboardUsageStatistics - inputFields - subTypes + - embed - name: notebook doc: Notebook represents a combination of query, text, chart and etc. This is in BETA version keyAspect: notebookKey diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index 75aa4ca300d61..7943a62d76b88 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -5,7 +5,7 @@ authentication: # Required if enabled is true! A configurable chain of Authenticators authenticators: - # Required for authenticating requests with DataHub-issued Access Tokens - best not to remove. + # Required for authenticating requests with DataHub-issued Access Tokens - best not to remove. - type: com.datahub.authentication.authenticator.DataHubTokenAuthenticator configs: # Key used to validate incoming tokens. Should typically be the same as authentication.tokenService.signingKey @@ -28,7 +28,6 @@ authentication: # Authorization-related configurations. authorization: - # Configurations for the default DataHub policies-based authorizer. defaultAuthorizer: enabled: ${AUTH_POLICIES_ENABLED:true} @@ -36,7 +35,7 @@ authorization: ingestion: enabled: ${UI_INGESTION_ENABLED:true} - defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}' + defaultCliVersion: "${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}" telemetry: enabledCli: ${CLI_TELEMETRY_ENABLED:true} @@ -92,6 +91,10 @@ platformAnalytics: enabled: ${DATAHUB_ANALYTICS_ENABLED:true} visualConfig: + queriesTab: + # Experimental! This env var is subject to change and may be deprecated in the future. The Queries tab has a larger + # overhaul coming. + queriesTabResultSize: ${REACT_APP_QUERIES_TAB_RESULT_SIZE:5} assets: logoUrl: ${REACT_APP_LOGO_URL:/assets/platforms/datahublogo.png} faviconUrl: ${REACT_APP_FAVICON_URL:/assets/favicon.ico} diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index 725702778527a..6f693f2f82f23 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -194,6 +194,11 @@ public class PoliciesConfig { "Edit Lineage", "The ability to add and remove lineage edges for this entity."); + public static final Privilege EDIT_ENTITY_EMBED_PRIVILEGE = Privilege.of( + "EDIT_ENTITY_EMBED", + "Edit Embedded Content", + "The ability to edit the embedded content for an entity."); + public static final List COMMON_ENTITY_PRIVILEGES = ImmutableList.of( VIEW_ENTITY_PAGE_PRIVILEGE, EDIT_ENTITY_TAGS_PRIVILEGE, @@ -279,7 +284,7 @@ public class PoliciesConfig { EDIT_DATASET_COL_TAGS_PRIVILEGE, EDIT_DATASET_COL_GLOSSARY_TERMS_PRIVILEGE, EDIT_ENTITY_ASSERTIONS_PRIVILEGE, - EDIT_LINEAGE_PRIVILEGE)) + EDIT_LINEAGE_PRIVILEGE, EDIT_ENTITY_EMBED_PRIVILEGE)) .flatMap(Collection::stream) .collect(Collectors.toList()) ); @@ -291,7 +296,7 @@ public class PoliciesConfig { "Charts indexed by DataHub", Stream.concat( COMMON_ENTITY_PRIVILEGES.stream(), - ImmutableList.of(EDIT_LINEAGE_PRIVILEGE).stream()) + ImmutableList.of(EDIT_LINEAGE_PRIVILEGE, EDIT_ENTITY_EMBED_PRIVILEGE).stream()) .collect(Collectors.toList()) ); @@ -302,7 +307,7 @@ public class PoliciesConfig { "Dashboards indexed by DataHub", Stream.concat( COMMON_ENTITY_PRIVILEGES.stream(), - ImmutableList.of(EDIT_LINEAGE_PRIVILEGE).stream()) + ImmutableList.of(EDIT_LINEAGE_PRIVILEGE, EDIT_ENTITY_EMBED_PRIVILEGE).stream()) .collect(Collectors.toList()) ); diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/EntityKeyUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/EntityKeyUtils.java index 5bab2014ba31b..2ad2d5028ba7d 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/EntityKeyUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/EntityKeyUtils.java @@ -93,7 +93,7 @@ public static RecordTemplate convertUrnToEntityKey(@Nonnull final Urn urn, @Nonn // #2. Bind fields into a DataMap if (urn.getEntityKey().getParts().size() != keySchema.getFields().size()) { throw new IllegalArgumentException( - "Failed to convert urn to entity key: urns parts and key fields do not have same length"); + "Failed to convert urn to entity key: urns parts and key fields do not have same length for " + urn); } final DataMap dataMap = new DataMap(); for (int i = 0; i < urn.getEntityKey().getParts().size(); i++) { diff --git a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py index d80b56b8cfcfb..f172e90469dcf 100644 --- a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py +++ b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py @@ -160,7 +160,7 @@ def _ensure_ingestion_source_present( if num_execs is not None: ingestion_source = res_data["data"]["ingestionSource"] - assert ingestion_source["executions"]["total"] > num_execs + assert ingestion_source["executions"]["total"] == num_execs return res_data @@ -327,7 +327,7 @@ def test_create_list_get_remove_ingestion_source(frontend_session): "name": "My Test Ingestion Source", "type": "mysql", "description": "My ingestion source description", - "schedule": {"interval": "*/5 * * * *", "timezone": "UTC"}, + "schedule": {"interval": "*/60 * * * *", "timezone": "UTC"}, "config": { "recipe": '{"source":{"type":"mysql","config":{"include_tables":true,"database":null,"password":"${MYSQL_PASSWORD}","profiling":{"enabled":false},"host_port":null,"include_views":true,"username":"${MYSQL_USERNAME}"}},"pipeline_name":"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927"}', "version": "0.8.18", @@ -389,7 +389,7 @@ def test_create_list_get_remove_ingestion_source(frontend_session): assert ingestion_source["urn"] == ingestion_source_urn assert ingestion_source["type"] == "mysql" assert ingestion_source["name"] == "My Test Ingestion Source" - assert ingestion_source["schedule"]["interval"] == "*/5 * * * *" + assert ingestion_source["schedule"]["interval"] == "*/60 * * * *" assert ingestion_source["schedule"]["timezone"] == "UTC" assert ( ingestion_source["config"]["recipe"]