Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(ingest): moving delta-lake connector to be 3.7+ only #5584

Merged
merged 4 commits into from
Aug 7, 2022
Merged

fix(ingest): moving delta-lake connector to be 3.7+ only #5584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0dd0069
fix(ingest): moving delta-lake connector to be 3.7+ only
shirshanka Aug 7, 2022
92aae90
move dbt to using third-party cached_property package
shirshanka Aug 7, 2022
fdd55bb
fix imports
shirshanka Aug 7, 2022
d13f9c2
skip tests
shirshanka Aug 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions metadata-ingestion/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def get_long_description():
"data-lake": {*data_lake_base, *data_lake_profiling},
"s3": {*s3_base, *data_lake_profiling},
"delta-lake": {*data_lake_profiling, *delta_lake},
"dbt": {"requests"} | aws_common,
"dbt": {"requests", "cached_property"} | aws_common,
"druid": sql_common | {"pydruid>=0.6.2"},
# Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws
# UnsupportedProductError
Expand Down Expand Up @@ -376,7 +376,6 @@ def get_long_description():
"bigquery-usage",
"clickhouse",
"clickhouse-usage",
"delta-lake",
"druid",
"elasticsearch",
"ldap",
Expand Down Expand Up @@ -415,6 +414,7 @@ def get_long_description():
{
dependency
for plugin in [
"delta-lake",
"feast",
"iceberg",
"lookml",
Expand All @@ -428,6 +428,7 @@ def get_long_description():
{
dependency
for plugin in [
"delta-lake",
"iceberg",
"lookml",
]
Expand Down Expand Up @@ -482,6 +483,7 @@ def get_long_description():
dependency
for plugin in [
"athena",
"delta-lake",
"feast",
"iceberg",
]
Expand Down
4 changes: 2 additions & 2 deletions metadata-ingestion/src/datahub/ingestion/source/dbt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from functools import cached_property
from typing import (
Any,
Callable,
Expand All @@ -21,6 +20,7 @@

import dateutil.parser
import requests
from cached_property import cached_property
from pydantic import BaseModel, root_validator, validator
from pydantic.fields import Field

Expand Down Expand Up @@ -192,7 +192,7 @@ def _any_other_only_set(self, attribute: str) -> bool:
]
return len(other_onlies) != 0

@cached_property
@cached_property # type: ignore
def node_type_emit_decision_cache(self) -> Dict[str, bool]:
node_type_for_field_map = {
"models": "model",
Expand Down
3 changes: 3 additions & 0 deletions metadata-ingestion/tests/integration/delta_lake/test_local_delta_lake.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import json
import logging
import os
import sys

import pytest

Expand All @@ -14,6 +15,7 @@


@pytest.mark.parametrize("source_file", source_files)
@pytest.mark.skipif(sys.version_info < (3, 7), reason="delta-lake requires Python 3.7+")
def test_delta_lake(pytestconfig, source_file, tmp_path, mock_time):
test_resources_dir = pytestconfig.rootpath / "tests/integration/delta_lake"

Expand Down Expand Up @@ -48,6 +50,7 @@ def test_delta_lake(pytestconfig, source_file, tmp_path, mock_time):
)


@pytest.mark.skipif(sys.version_info < (3, 7), reason="delta-lake requires Python 3.7+")
def test_data_lake_incorrect_config_raises_error(tmp_path, mock_time):
config_dict = {}
config_dict["sink"] = {
Expand Down