Merge branch 'master' into ab-fix-invalid-urls

datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx

@@ -7,6 +7,7 @@ import analytics, { EventType } from '../../../analytics';
 import { useGlossaryEntityData } from '../GlossaryEntityContext';
 import { getParentNodeToUpdate, updateGlossarySidebar } from '../../../glossary/utils';
 import { useHandleDeleteDomain } from './useHandleDeleteDomain';
+import { removeTermFromGlossaryNode } from '../../../glossary/cacheUtils';
 
 /**
  * Performs the flow for deleting an entity of a given type.
@@ -30,6 +31,7 @@ function useDeleteEntity(
 
     const maybeDeleteEntity = getDeleteEntityMutation(type)();
     const deleteEntity = (maybeDeleteEntity && maybeDeleteEntity[0]) || undefined;
+    const client = maybeDeleteEntity?.[1].client;
 
     function handleDeleteEntity() {
         deleteEntity?.({
@@ -54,6 +56,10 @@ function useDeleteEntity(
                     handleDeleteDomain();
                 }
 
+                if (client && entityData.type === EntityType.GlossaryTerm && entityData?.parentNodes?.nodes) {
+                    removeTermFromGlossaryNode(client, entityData.parentNodes.nodes[0].urn, urn);
+                }
+
                 setTimeout(
                     () => {
                         setHasBeenDeleted(true);

datahub-web-react/src/app/glossary/cacheUtils.ts

@@ -0,0 +1,36 @@
+import { ApolloClient } from '@apollo/client';
+import { GetGlossaryNodeDocument, GetGlossaryNodeQuery } from '../../graphql/glossaryNode.generated';
+
+export function removeTermFromGlossaryNode(
+    client: ApolloClient<object>,
+    glossaryNodeUrn: string,
+    glossaryTermUrn: string,
+) {
+    // Read the data from our cache for this query.
+    const currData: GetGlossaryNodeQuery | null = client.readQuery({
+        query: GetGlossaryNodeDocument,
+        variables: { urn: glossaryNodeUrn },
+    });
+
+    // Remove the term from the existing children set.
+    const newTermChildren = {
+        relationships: [
+            ...(currData?.glossaryNode?.children?.relationships || []).filter(
+                (relationship) => relationship.entity?.urn !== glossaryTermUrn,
+            ),
+        ],
+        total: (currData?.glossaryNode?.children?.total || 1) - 1,
+    };
+
+    // Write our data back to the cache.
+    client.writeQuery({
+        query: GetGlossaryNodeDocument,
+        variables: { urn: glossaryNodeUrn },
+        data: {
+            glossaryNode: {
+                ...currData?.glossaryNode,
+                children: newTermChildren,
+            },
+        },
+    });
+}

docker/datahub-frontend/Dockerfile

@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM alpine:3 AS base
+FROM alpine:3.18 AS base
 
 # Configurable repositories
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine

docker/datahub-gms/Dockerfile

@@ -24,7 +24,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3 AS base
+FROM alpine:3.18 AS base
 
 # Upgrade Alpine and base packages
 ENV JMX_VERSION=0.18.0

docker/datahub-mae-consumer/Dockerfile

@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3 AS base
+FROM alpine:3.18 AS base
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL

docker/datahub-mce-consumer/Dockerfile

@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3 AS base
+FROM alpine:3.18 AS base
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL

docker/datahub-upgrade/Dockerfile

@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3 AS base
+FROM alpine:3.18 AS base
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL

docker/elasticsearch-setup/Dockerfile

@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3 AS base
+FROM alpine:3.18 AS base
 
 ARG ALPINE_REPO_URL
 

docker/mysql-setup/Dockerfile

@@ -17,7 +17,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3
+FROM alpine:3.18
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
 ARG ALPINE_REPO_URL

docker/postgres-setup/Dockerfile

@@ -17,7 +17,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
 
 RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 
-FROM alpine:3
+FROM alpine:3.18
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
 ARG ALPINE_REPO_URL

docs/modeling/extending-the-metadata-model.md

@@ -256,7 +256,7 @@ to deploy during development. This will allow Datahub to read and write your new
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-<Tabs>
+<Tabs queryString="python-custom-models">
 <TabItem value="local" label="Local CLI" default>
 
 If you're purely using the custom models locally, you can use a local development-mode install of the DataHub CLI.
@@ -273,12 +273,21 @@ If you want to use your custom models beyond your local machine without forking
 This package should be installed alongside the base `acryl-datahub` package, and its metadata models will take precedence over the default ones.
 
 ```bash
-cd metadata-ingestion
-../gradlew customPackageGenerate -Ppackage_name=my-company-datahub-models -Ppackage_version="0.0.1"
+$ cd metadata-ingestion
+$ ../gradlew customPackageGenerate -Ppackage_name=my-company-datahub-models -Ppackage_version="0.0.1"
+<bunch of log lines>
+Successfully built my-company-datahub-models-0.0.1.tar.gz and acryl_datahub_cloud-0.0.1-py3-none-any.whl
+
+Generated package at custom-package/my-company-datahub-models
+This package should be installed alongside the main acryl-datahub package.
+
+Install the custom package locally with `pip install custom-package/my-company-datahub-models`
+To enable others to use it, share the file at custom-package/my-company-datahub-models/dist/<wheel file>.whl and have them install it with `pip install <wheel file>.whl`
+Alternatively, publish it to PyPI with `twine upload custom-package/my-company-datahub-models/dist/*`
 ```
 
 This will generate some Python build artifacts, which you can distribute within your team or publish to PyPI.
-The command output will contain additional details and exact CLI commands you can use.
+The command output contains additional details and exact CLI commands you can use.
 
 </TabItem>
 </Tabs>

metadata-ingestion/build.gradle

@@ -57,7 +57,10 @@ task installPackage(type: Exec, dependsOn: installPackageOnly) {
 }
 
 task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
-  inputs.files(project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"))
+  inputs.files(
+    project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"),
+    project.fileTree(dir: "scripts"),
+  )
   outputs.dir('src/datahub/metadata')
   commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/codegen.sh"
 }

metadata-ingestion/scripts/avro_codegen.py

@@ -252,34 +252,12 @@ def annotate_aspects(aspects: List[dict], schema_class_file: Path) -> None:
     schema_classes_lines = schema_class_file.read_text().splitlines()
     line_lookup_table = {line: i for i, line in enumerate(schema_classes_lines)}
 
-    # Create the Aspect class.
-    # We ensure that it cannot be instantiated directly, as
-    # per https://stackoverflow.com/a/7989101/5004662.
+    # Import the _Aspect class.
     schema_classes_lines[
         line_lookup_table["__SCHEMAS: Dict[str, RecordSchema] = {}"]
     ] += """
 
-class _Aspect(DictWrapper):
-    ASPECT_NAME: ClassVar[str] = None  # type: ignore
-    ASPECT_TYPE: ClassVar[str] = "default"
-    ASPECT_INFO: ClassVar[dict] = None  # type: ignore
-
-    def __init__(self):
-        if type(self) is _Aspect:
-            raise TypeError("_Aspect is an abstract class, and cannot be instantiated directly.")
-        super().__init__()
-
-    @classmethod
-    def get_aspect_name(cls) -> str:
-        return cls.ASPECT_NAME  # type: ignore
-
-    @classmethod
-    def get_aspect_type(cls) -> str:
-        return cls.ASPECT_TYPE
-
-    @classmethod
-    def get_aspect_info(cls) -> dict:
-        return cls.ASPECT_INFO
+from datahub._codegen.aspect import _Aspect
 """
 
     for aspect in aspects:
@@ -776,6 +754,7 @@ def generate(
 import importlib
 from typing import TYPE_CHECKING
 
+from datahub._codegen.aspect import _Aspect
 from datahub.utilities.docs_build import IS_SPHINX_BUILD
 from datahub.utilities._custom_package_loader import get_custom_models_package
 
@@ -785,7 +764,7 @@ def generate(
     from ._schema_classes import *
 
     # Required explicitly because __all__ doesn't include _ prefixed names.
-    from ._schema_classes import _Aspect, __SCHEMA_TYPES
+    from ._schema_classes import __SCHEMA_TYPES
 
     if IS_SPHINX_BUILD:
         # Set __module__ to the current module so that Sphinx will document the

metadata-ingestion/scripts/custom_package_codegen.py

@@ -73,6 +73,8 @@ def generate(
 """
     )
 
+    (src_path / "py.typed").write_text("")
+
     (package_path / "setup.py").write_text(
         f"""{autogen_header}
 from setuptools import setup
@@ -87,6 +89,11 @@ def generate(
         "avro-gen3=={_avrogen_version}",
         "acryl-datahub",
     ],
+    package_data={{
+        "{python_package_name}": ["py.typed"],
+        "{python_package_name}.models": ["schema.avsc"],
+        "{python_package_name}.models.schemas": ["*.avsc"],
+    }},
     entry_points={{
         "datahub.custom_packages": [
             "models={python_package_name}.models.schema_classes",
@@ -109,7 +116,9 @@ def generate(
     click.echo()
     click.echo(f"Install the custom package locally with `pip install {package_path}`")
     click.echo(
-        f"To enable others to use it, share the file at {package_path}/dist/*.whl and have them install it with `pip install <wheel file>.whl`"
+        "To enable others to use it, share the file at "
+        f"{package_path}/dist/{package_name}-{package_version}-py3-none-any.whl "
+        "and have them install it with `pip install <wheel file>.whl`"
     )
     click.echo(
         f"Alternatively, publish it to PyPI with `twine upload {package_path}/dist/*`"

metadata-ingestion/setup.py

@@ -650,7 +650,7 @@
         "simple_add_dataset_properties = datahub.ingestion.transformer.add_dataset_properties:SimpleAddDatasetProperties",
         "pattern_add_dataset_schema_terms = datahub.ingestion.transformer.add_dataset_schema_terms:PatternAddDatasetSchemaTerms",
         "pattern_add_dataset_schema_tags = datahub.ingestion.transformer.add_dataset_schema_tags:PatternAddDatasetSchemaTags",
-        "extract_owners_from_tags = datahub.ingestion.transformer.extract_ownership_from_tags:ExtractOwnersFromTagsTransformer",
+        "extract_ownership_from_tags = datahub.ingestion.transformer.extract_ownership_from_tags:ExtractOwnersFromTagsTransformer",
     ],
     "datahub.ingestion.sink.plugins": [
         "file = datahub.ingestion.sink.file:FileSink",

metadata-ingestion/src/datahub/_codegen/aspect.py

@@ -0,0 +1,36 @@
+from typing import ClassVar
+
+from avrogen.dict_wrapper import DictWrapper
+
+
+class _Aspect(DictWrapper):
+    """Base class for all aspects types.
+
+    All codegened types inherit from DictWrapper, either directly or indirectly.
+    Types that are aspects inherit directly from _Aspect.
+    """
+
+    ASPECT_NAME: ClassVar[str] = None  # type: ignore
+    ASPECT_TYPE: ClassVar[str] = "default"
+    ASPECT_INFO: ClassVar[dict] = None  # type: ignore
+
+    def __init__(self):
+        if type(self) is _Aspect:
+            # Ensure that it cannot be instantiated directly, as
+            # per https://stackoverflow.com/a/7989101/5004662.
+            raise TypeError(
+                "_Aspect is an abstract class, and cannot be instantiated directly."
+            )
+        super().__init__()
+
+    @classmethod
+    def get_aspect_name(cls) -> str:
+        return cls.ASPECT_NAME  # type: ignore
+
+    @classmethod
+    def get_aspect_type(cls) -> str:
+        return cls.ASPECT_TYPE
+
+    @classmethod
+    def get_aspect_info(cls) -> dict:
+        return cls.ASPECT_INFO

metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py

@@ -1315,8 +1315,6 @@ def get_schema_metadata(
             self.config.strip_user_ids_from_email,
         )
 
-        # TODO if infer_dbt_schemas, load from saved schemas too
-
         canonical_schema: List[SchemaField] = []
         for column in node.columns:
             description = None

metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py

@@ -466,6 +466,19 @@ def load_nodes(self) -> Tuple[List[DBTNode], Dict[str, Optional[str]]]:
             catalog_version,
         ) = self.loadManifestAndCatalog()
 
+        # If catalog_version is between 1.7.0 and 1.7.2, report a warning.
+        if (
+            catalog_version
+            and catalog_version.startswith("1.7.")
+            and catalog_version < "1.7.3"
+        ):
+            self.report.report_warning(
+                "dbt_catalog_version",
+                f"Due to a bug in dbt, dbt version {catalog_version} will have incomplete metadata on sources. "
+                "Please upgrade to dbt version 1.7.3 or later. "
+                "See https://github.com/dbt-labs/dbt-core/issues/9119 for details on the bug.",
+            )
+
         additional_custom_props = {
             "manifest_schema": manifest_schema,
             "manifest_version": manifest_version,

metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py

@@ -157,7 +157,7 @@ def __getattr__(self, item: str) -> Any:
 
 @platform_name("Oracle")
 @config_class(OracleConfig)
-@support_status(SupportStatus.CERTIFIED)
+@support_status(SupportStatus.INCUBATING)
 @capability(SourceCapability.DOMAINS, "Enabled by default")
 class OracleSource(SQLAlchemySource):
     """

metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json

@@ -865,9 +865,6 @@
                 }
             },
             "inputs": [
-                {
-                    "string": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
-                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
                 },

smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js

@@ -1,6 +1,6 @@
 const glossaryTerm = "CypressGlosssaryNavigationTerm";
 const glossaryTermGroup = "CypressGlosssaryNavigationGroup";
-const glossaryParentGroup = "Cypress";
+const glossaryParentGroup = "CypressNode";
 
 describe("glossary sidebar navigation test", () => {
     it("create term and term parent group, move and delete term group", () => {
@@ -33,6 +33,7 @@ describe("glossary sidebar navigation test", () => {
         // Move a term group from the root level to be under a parent term group
         cy.goToGlossaryList();
         cy.clickOptionWithText(glossaryTermGroup);
+        cy.wait(3000)
         cy.openThreeDotDropdown();
         cy.clickOptionWithText("Move");
         cy.get('[data-testid="move-glossary-entity-modal"]').contains(glossaryParentGroup).click({force: true});