Merge branch 'devel' into feat/destination-registry

.github/workflows/lint.yml

@@ -26,7 +26,7 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-        python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
+        python-version: ["3.9.x", "3.10.x", "3.11.x"]
 
     defaults:
       run:
@@ -49,6 +49,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_airflow.yml

@@ -37,6 +37,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_build_images.yml

@@ -37,6 +37,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Build images
         run: make test-build-images

.github/workflows/test_common.yml

@@ -37,8 +37,6 @@ jobs:
         python-version: ["3.11.x"]
         # Test all python versions on ubuntu only
         include:
-          - python-version: "3.8.x"
-            os: "ubuntu-latest"
           - python-version: "3.9.x"
             os: "ubuntu-latest"
           - python-version: "3.10.x"
@@ -79,6 +77,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
       # - name: Load cached venv

.github/workflows/test_dbt_cloud.yml

@@ -52,6 +52,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_dbt_runner.yml

@@ -49,6 +49,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_athena.yml

@@ -56,6 +56,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_athena_iceberg.yml

@@ -56,6 +56,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_bigquery.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_clickhouse.yml

@@ -52,6 +52,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_databricks.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_dremio.yml

@@ -56,6 +56,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_lancedb.yml

@@ -53,6 +53,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_motherduck.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_mssql.yml

@@ -60,6 +60,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_qdrant.yml

@@ -54,6 +54,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_snowflake.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_synapse.yml

@@ -58,6 +58,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destinations.yml

@@ -67,6 +67,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       # - name: Load cached venv
       #   id: cached-poetry-dependencies

.github/workflows/test_doc_snippets.yml

@@ -78,6 +78,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_local_destinations.yml

@@ -86,6 +86,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_local_sources.yml

@@ -73,6 +73,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_sqlalchemy_destinations.yml

@@ -77,6 +77,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

dlt/cli/pipeline_files.py

@@ -4,11 +4,12 @@
 import yaml
 import posixpath
 from pathlib import Path
-from typing import Dict, NamedTuple, Sequence, Tuple, TypedDict, List, Literal
+from typing import Dict, NamedTuple, Sequence, Tuple, List, Literal
 from dlt.cli.exceptions import VerifiedSourceRepoError
 
 from dlt.common import git
 from dlt.common.storages import FileStorage
+from dlt.common.typing import TypedDict
 
 from dlt.common.reflection.utils import get_module_docstring
 

dlt/common/exceptions.py

@@ -1,4 +1,5 @@
-from typing import Any, AnyStr, Dict, List, Sequence, Optional, Iterable, Type, TypedDict
+from typing import Any, AnyStr, Dict, List, Sequence, Optional, Type
+from dlt.common.typing import TypedDict
 
 
 class ExceptionTrace(TypedDict, total=False):

dlt/common/incremental/typing.py

@@ -1,8 +1,6 @@
-from typing_extensions import TypedDict
-
 from typing import Any, Callable, List, Literal, Optional, Sequence, TypeVar, Union
 
-from dlt.common.typing import TSortOrder, TTableHintTemplate, TColumnNames
+from dlt.common.typing import TSortOrder, TTableHintTemplate, TColumnNames, TypedDict
 
 TCursorValue = TypeVar("TCursorValue", bound=Any)
 LastValueFunc = Callable[[Sequence[TCursorValue]], Any]

dlt/common/libs/pyarrow.py

@@ -462,16 +462,8 @@ def to_arrow_scalar(value: Any, arrow_type: pyarrow.DataType) -> Any:
 
 
 def from_arrow_scalar(arrow_value: pyarrow.Scalar) -> Any:
-    """Converts arrow scalar into Python type. Currently adds "UTC" to naive date times and converts all others to UTC"""
-    row_value = arrow_value.as_py()
-    # dates are not represented as datetimes but I see connector-x represents
-    # datetimes as dates and keeping the exact time inside. probably a bug
-    # but can be corrected this way
-    if isinstance(row_value, date) and not isinstance(row_value, datetime):
-        row_value = pendulum.from_timestamp(arrow_value.cast(pyarrow.int64()).as_py() / 1000)
-    elif isinstance(row_value, datetime):
-        row_value = pendulum.instance(row_value).in_tz("UTC")
-    return row_value
+    """Converts arrow scalar into Python type."""
+    return arrow_value.as_py()
 
 
 TNewColumns = Sequence[Tuple[int, pyarrow.Field, Callable[[pyarrow.Table], Iterable[Any]]]]

dlt/common/libs/pydantic.py

@@ -6,7 +6,6 @@
     Generic,
     Optional,
     Set,
-    TypedDict,
     List,
     Type,
     Union,
@@ -15,6 +14,7 @@
 )
 from typing_extensions import Annotated, get_args, get_origin
 
+from dlt.common.typing import TypedDict
 from dlt.common.data_types import py_type_to_sc_type
 from dlt.common.exceptions import MissingDependencyException
 from dlt.common.schema import DataValidationError

dlt/common/metrics.py

@@ -1,5 +1,6 @@
 import datetime  # noqa: I251
-from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict  # noqa: 251
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple  # noqa: 251
+from dlt.common.typing import TypedDict
 
 
 class DataWriterMetrics(NamedTuple):

dlt/common/normalizers/json/relational.py

@@ -8,10 +8,10 @@
     Tuple,
     Type,
     cast,
-    TypedDict,
     Any,
 )
 
+from dlt.common.typing import TypedDict
 from dlt.common.normalizers.exceptions import InvalidJsonNormalizer
 from dlt.common.normalizers.typing import TJSONNormalizer
 from dlt.common.normalizers.utils import generate_dlt_id

dlt/common/normalizers/typing.py

@@ -1,7 +1,7 @@
-from typing import List, Optional, Type, TypedDict, Literal, Union
+from typing import List, Optional, Type, Literal, Union
 from types import ModuleType
 
-from dlt.common.typing import StrAny
+from dlt.common.typing import StrAny, TypedDict
 from dlt.common.normalizers.naming import NamingConvention
 
 TNamingConventionReferenceArg = Union[str, Type[NamingConvention], ModuleType]

dlt/common/pipeline.py

@@ -18,12 +18,12 @@
     Sequence,
     Tuple,
     TypeVar,
-    TypedDict,
     Mapping,
     Literal,
 )
 from typing_extensions import NotRequired
 
+from dlt.common.typing import TypedDict
 from dlt.common.configuration import configspec
 from dlt.common.configuration import known_sections
 from dlt.common.configuration.container import Container

dlt/common/runtime/typing.py

@@ -1,9 +1,8 @@
 from typing import (
     List,
     Literal,
-    TypedDict,
 )
-
+from dlt.common.typing import TypedDict
 
 TExecInfoNames = Literal[
     "kubernetes",

dlt/common/schema/typing.py

@@ -10,13 +10,13 @@
     Set,
     Tuple,
     Type,
-    TypedDict,
     NewType,
     Union,
     get_args,
 )
 from typing_extensions import Never
 
+from dlt.common.typing import TypedDict
 from dlt.common.data_types import TDataType
 from dlt.common.normalizers.typing import TNormalizersConfig
 from dlt.common.typing import TSortOrder, TAnyDateTime, TLoaderFileFormat, TColumnNames

dlt/common/storages/fsspec_filesystem.py

@@ -9,7 +9,6 @@
     Literal,
     cast,
     Tuple,
-    TypedDict,
     Optional,
     Union,
     Iterator,
@@ -25,6 +24,7 @@
 from fsspec.core import url_to_fs
 
 from dlt import version
+from dlt.common.typing import TypedDict
 from dlt.common.pendulum import pendulum
 from dlt.common.configuration.specs import (
     GcpCredentials,

dlt/common/storages/load_package.py

@@ -21,10 +21,10 @@
     cast,
     Any,
     Tuple,
-    TypedDict,
 )
 from typing_extensions import NotRequired
 
+from dlt.common.typing import TypedDict
 from dlt.common.pendulum import pendulum
 from dlt.common.json import json
 from dlt.common.configuration import configspec

dlt/common/typing.py

@@ -45,6 +45,8 @@
 
 from typing_extensions import is_typeddict as _is_typeddict
 
+from typing_extensions import TypedDict  # noqa: I251
+
 try:
     from types import UnionType  # type: ignore[attr-defined]
 except ImportError:

dlt/common/versioned_state.py

@@ -2,11 +2,12 @@
 import hashlib
 import binascii
 from copy import copy
-from typing import TypedDict, List, Tuple, Mapping
+from typing import List, Tuple, Mapping
 
 from dlt.common.json import json
 from dlt.common.typing import DictStrAny
 from dlt.common.utils import compressed_b64decode, compressed_b64encode
+from dlt.common.typing import TypedDict
 
 
 class TVersionedState(TypedDict, total=False):