Merge branch 'main' into full-path-error

.github/workflows/constraints.txt

@@ -1,6 +1,5 @@
 pip==22.1.2
-poetry==1.1.13
+poetry==1.1.14
 virtualenv==20.15.1
-changelog-cli==0.7.1
 nox==2022.1.7
 nox-poetry==1.0.1

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
       uses: actions/[email protected]
 
     - name: Set up Python
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: "3.10"
 

.github/workflows/test.yml

@@ -25,6 +25,14 @@ jobs:
         - { session: tests,   python-version: "3.8",  os: "ubuntu-latest" }
         - { session: tests,   python-version: "3.9",  os: "ubuntu-latest" }
         - { session: tests,   python-version: "3.10", os: "ubuntu-latest" }
+        - { session: tests,   python-version: "3.7",  os: "macos-latest" }
+        - { session: tests,   python-version: "3.8",  os: "macos-latest" }
+        - { session: tests,   python-version: "3.9",  os: "macos-latest" }
+        - { session: tests,   python-version: "3.10", os: "macos-latest" }
+        - { session: tests,   python-version: "3.7",  os: "windows-latest" }
+        - { session: tests,   python-version: "3.8",  os: "windows-latest" }
+        - { session: tests,   python-version: "3.9",  os: "windows-latest" }
+        - { session: tests,   python-version: "3.10", os: "windows-latest" }
         - { session: doctest, python-version: "3.10", os: "ubuntu-latest" }
         - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest" }
 
@@ -40,7 +48,7 @@ jobs:
         poetry --version
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64
@@ -98,7 +106,7 @@ jobs:
         poetry --version
 
     - name: Setup Python 3.10
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: '3.10'
         architecture: x64
@@ -138,7 +146,7 @@ jobs:
         poetry --version
 
     - name: Set up Python
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: '3.10'
         cache: 'pip'

.github/workflows/version_bump.yml

@@ -7,11 +7,12 @@ on:
         description: "Version bump type"
         required: true
         type: choice
-        default: PATCH
+        default: auto
         options:
-        - PATCH
-        - MINOR
-        - MAJOR
+        - auto
+        - patch
+        - minor
+        - major
       prerelease:
         description: "Increase to this prerelease version"
         required: false
@@ -39,17 +40,16 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: "3.10"
         architecture: x64
 
     - name: Bump version
       id: cz-bump
-      # TODO: https://github.com/commitizen-tools/commitizen-action/pull/38
-      uses: edgarrmondragon/commitizen-action@feat-add-increment-option
+      uses: commitizen-tools/[email protected]
       with:
-        increment: ${{ github.event.inputs.bump }}
+        increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }}
         prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }}
         commit: "false"
         push: "false"
@@ -88,3 +88,4 @@ jobs:
         base: main
         labels: release
         assignees: "${{ github.actor }}"
+        token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
         )$
 
 - repo: https://github.com/psf/black
-  rev: 22.3.0
+  rev: 22.6.0
   hooks:
   - id: black
     language_version: python3.10

README.md

@@ -36,6 +36,8 @@ to get started with the SDK.
    $ gh workflow run
    ```
 
+   The `increment: auto` option will figure out the most appropriate bump based on commit history.
+
 1. Follow the checklist in the PR description.
 
 1. Publish a new release [using the GitHub web UI](https://github.com/meltano/sdk/releases/new).

pyproject.toml

@@ -56,13 +56,11 @@ docs = ["sphinx", "sphinx-rtd-theme", "sphinx-copybutton", "myst-parser"]
 [tool.poetry.dev-dependencies]
 # snowflake-connector-python = "2.0.4" # Removed: Too many version conflicts!
 pytest = "^7.1.2"
-xdoctest = "^1.0.0"
+xdoctest = "^1.0.1"
 mypy = "^0.961"
 cookiecutter = "^2.1.1"
 PyYAML = "^6.0"
 pyarrow = "^8.0.0"
-pandas = ">=1.2.0"
-numpy = ">=1.20.0"
 freezegun = "^1.2.1"
 viztracer = "^0.15.3"
 requests-mock = "^1.9.3"
@@ -104,8 +102,19 @@ version_files = [
     "cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml:singer-sdk",
 ]
 
+[tool.coverage.paths]
+source = [
+    "singer_sdk/",
+    "*/singer_sdk",
+]
+tests = [
+    "tests/",
+    "*/tests",
+]
+
 [tool.coverage.run]
-source = ["singer_sdk"]
+branch = true
+source = ["singer_sdk", "tests"]
 omit = [
     "tests/*",
     "samples/*",
@@ -121,7 +130,7 @@ exclude_lines = [
     "if __name__ == .__main__.:",
     '''class .*\bProtocol\):''',
 ]
-fail_under = 85
+fail_under = 82
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

samples/sample_target_parquet/parquet_target.py

@@ -14,3 +14,7 @@ class SampleTargetParquet(Target):
         th.Property("file_naming_scheme", th.StringType),
     ).to_dict()
     default_sink_class = SampleParquetTargetSink
+
+
+if __name__ == "__main__":
+    SampleTargetParquet.cli()

samples/sample_target_parquet/parquet_target_sink.py

@@ -1,14 +1,88 @@
 """Sample Parquet target stream class, which handles writing streams."""
 
-from typing import Any, Dict, List, Tuple, Union
+from __future__ import annotations
+
+from typing import Any, Dict, Union
 
-import pandas
 import pyarrow as pa
 import pyarrow.parquet as pq
 
 from singer_sdk.sinks import BatchSink
 
 
+def json_schema_to_arrow(schema: dict[str, Any]) -> pa.Schema:
+    """Convert a JSON Schema to an Arrow schema.
+
+    Args:
+        schema: The JSON Schema definition.
+
+    Returns:
+        An Arrow schema.
+    """
+    fields = _json_schema_to_arrow_fields(schema)
+    return pa.schema(fields)
+
+
+def _json_schema_to_arrow_fields(schema: dict[str, Any]) -> pa.StructType:
+    """Convert a JSON Schema to an Arrow struct.
+
+    Args:
+        schema: The JSON Schema definition.
+
+    Returns:
+        An Arrow struct.
+    """
+    fields = []
+    for name, property_schema in schema.get("properties", {}).items():
+        field = pa.field(name, _json_type_to_arrow_field(property_schema))
+        fields.append(field)
+    return fields
+
+
+def _json_type_to_arrow_field(schema_type: dict[str, Any]) -> pa.DataType:
+    """Convert a JSON Schema to an Arrow struct.
+
+    Args:
+        schema: The JSON Schema definition.
+
+    Returns:
+        An Arrow struct.
+    """
+    property_type = schema_type.get("type")
+
+    if isinstance(property_type, list):
+        try:
+            main_type = property_type[0]
+        except IndexError:
+            main_type = "null"
+    else:
+        main_type = property_type
+
+    if main_type == "array":
+        items = schema_type.get("items", {})
+        return pa.list_(_json_type_to_arrow_field(items))
+
+    elif main_type == "object":
+        return pa.struct(_json_schema_to_arrow_fields(schema_type))
+
+    elif main_type == "string":
+        return pa.string()
+
+    elif main_type == "integer":
+        return pa.int64()
+
+    elif main_type == "number":
+        return pa.float64()
+
+    elif main_type == "boolean":
+        return pa.bool_()
+
+    elif main_type == "null":
+        return pa.null()
+
+    return pa.null()
+
+
 class SampleParquetTargetSink(BatchSink):
     """Parquery target sample class."""
 
@@ -17,17 +91,15 @@ class SampleParquetTargetSink(BatchSink):
     def process_batch(self, context: dict) -> None:
         """Write any prepped records out and return only once fully written."""
         records_to_drain = context["records"]
-        # TODO: Replace with actual schema from the SCHEMA message
-        schema = pa.schema([("some_int", pa.int32()), ("some_string", pa.string())])
+        schema = json_schema_to_arrow(self.schema)
         writer = pq.ParquetWriter(self.config["filepath"], schema)
 
-        df = pandas.DataFrame(data=records_to_drain)
-        table = pa.Table.from_pandas(df)
+        table = pa.Table.from_pylist(records_to_drain, schema=schema)
         writer.write_table(table)
         writer.close()
 
     @staticmethod
-    def translate_data_type(singer_type: Union[str, Dict]) -> Any:
+    def translate_data_type(singer_type: str | dict) -> Any:
         """Translate from singer_type to a native type."""
         if singer_type in ["decimal", "float", "double"]:
             return pa.decimal128

singer_sdk/typing.py

@@ -86,7 +86,15 @@
     "PropertiesList",
 ]
 
-_JsonValue: TypeAlias = Union[str, int, float, bool, None]
+_JsonValue: TypeAlias = Union[
+    str,
+    int,
+    float,
+    bool,
+    list,
+    dict,
+    None,
+]
 
 
 def extend_validator_with_defaults(validator_class):  # noqa

tests/core/test_capabilities.py

@@ -1,3 +1,4 @@
+import warnings
 from inspect import currentframe, getframeinfo
 
 import pytest
@@ -13,11 +14,10 @@ class DummyCapabilitiesEnum(CapabilitiesEnum):
 
 
 def test_deprecated_capabilities():
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
         DummyCapabilitiesEnum.MY_SUPPORTED_FEATURE
 
-    assert len(record.list) == 0
-
     with pytest.warns(
         DeprecationWarning,
         match="is deprecated. No longer supported",