Merge branch 'main' into edgarrmondragon/refactor/msgspec-impl-naive

meltano · Sep 6, 2024 · 08b58bf · 08b58bf
2 parents 6caf27d + 4ce312f
commit 08b58bf
Show file tree

Hide file tree

Showing 59 changed files with 1,470 additions and 637 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -13,7 +13,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.39.1"
+      placeholder: "0.40.0"
     validations:
       required: true
   - type: checkboxes
@@ -29,13 +29,12 @@ body:
       label: Python Version
       description: Version of Python you are using
       options:
-        - "3.6 (EOL)"
-        - "3.7 (EOL)"
-        - "3.8"
-        - "3.9"
-        - "3.10"
-        - "3.11"
         - "3.12"
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        - "3.8"
+        - "3.7 or earlier"
         - "NA"
     validations:
       required: true
@@ -78,3 +77,9 @@ body:
       render: Python
     validations:
       required: false
+  - type: input
+    id: slack_or_linen
+    attributes:
+      label: Link to Slack/Linen
+      description: Provide a link to the Slack or Linen conversation, if applicable
+      placeholder: "https://..."
diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,4 +1,4 @@
-griffe==0.48.0
+griffe==1.2.0
 pip==24.2
 poetry==1.8.3
 poetry-plugin-export==1.8.0

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -5,15 +5,52 @@ on:
 
 jobs:
   build:
+    name: Build artifacts
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - uses: hynek/build-and-inspect-python-package@v2
 
+  check-tag:
+    name: Check tag
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    outputs:
+      is_final: ${{ steps.check.outputs.is_final }}
+    steps:
+    - name: Check if tag is a pre-release
+      id: check
+      run: |
+        echo "is_final=$(echo '${{ github.ref }}' | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+$' && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT
+
+  provenance:
+    name: Provenance
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: startsWith(github.ref, 'refs/tags/')
+    permissions:
+      id-token: write     # Needed for attestations
+      attestations: write # Needed for attestations
+    outputs:
+      bundle-path: ${{ steps.attest.outputs.bundle-path }}
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+    - uses: actions/attest-build-provenance@v1
+      id: attest
+      with:
+        subject-path: "./dist/singer_sdk*"
+    - uses: actions/upload-artifact@v4
+      with:
+        name: Attestations
+        path: ${{ steps.attest.outputs.bundle-path }}
+
   publish:
-    name: Publish to PyPI
+    name: PyPI
     runs-on: ubuntu-latest
     needs: [build]
     environment:
@@ -28,38 +65,39 @@ jobs:
         name: Packages
         path: dist
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@v1.9.0
+      uses: pypa/gh-action-pypi-publish@v1.10.0
 
   upload-to-release:
     name: Upload files to release
     runs-on: ubuntu-latest
-    needs: [build]
-    if: startsWith(github.ref, 'refs/tags/')
+    needs: [build, check-tag, provenance]
+    if: ${{ startsWith(github.ref, 'refs/tags/') && needs.check-tag.outputs.is_final == 'true' }}
     permissions:
       contents: write     # Needed for uploading files to the release
-      id-token: write     # Needed for attestations
-      attestations: write # Needed for attestations
 
     steps:
     - uses: actions/download-artifact@v4
       with:
         name: Packages
         path: dist
+
+    - uses: actions/download-artifact@v4
+      with:
+        name: Attestations
+        path: attestations
+
     - name: Upload wheel and sdist to release
       uses: svenstaro/upload-release-action@v2
       with:
         file: dist/singer_sdk*
         tag: ${{ github.ref }}
         overwrite: true
         file_glob: true
-    - uses: actions/attest-build-provenance@v1
-      id: attest
-      with:
-        subject-path: "./dist/singer_sdk*"
+
     - name: Upload attestations to release
       uses: svenstaro/upload-release-action@v2
       with:
-        file: ${{ steps.attest.outputs.bundle-path }}
+        file: attestations/attestation.jsonl
         tag: ${{ github.ref }}
         overwrite: true
         asset_name: attestations.intoto.jsonl
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -99,6 +99,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always() && (matrix.session == 'tests')
       with:
+        include-hidden-files: true
         name: coverage-data-nox_${{ matrix.session }}-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }}
         path: ".coverage.*"
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -43,14 +43,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.7
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,40 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.40.0 (2024-09-02)
+
+### ✨ New
+
+- [#2486](https://github.com/meltano/sdk/issues/2486) Emit target metrics
+- [#2567](https://github.com/meltano/sdk/issues/2567) A new `schema_is_valid` built-in tap test validates stream schemas against the JSON Schema specification
+- [#2598](https://github.com/meltano/sdk/issues/2598) Stream map expressions now have access to the `Faker` class, rather than just a faker instance
+- [#2549](https://github.com/meltano/sdk/issues/2549) Added a default user agent for REST and GraphQL taps
+
+### 🐛 Fixes
+
+- [#2613](https://github.com/meltano/sdk/issues/2613) Mismatch between timezone-aware and naive datetimes in start date and bookmarks is now correctly handled
+
+### ⚙️ Under the Hood
+
+- [#2628](https://github.com/meltano/sdk/issues/2628) Use context manager to read gzip batch files
+- [#2619](https://github.com/meltano/sdk/issues/2619) Default to UTC when parsing dates without a known timezone
+- [#2603](https://github.com/meltano/sdk/issues/2603) Backwards-compatible identifier quoting in fully qualified names
+- [#2601](https://github.com/meltano/sdk/issues/2601) Improved SQL identifier (de)normalization
+- [#2599](https://github.com/meltano/sdk/issues/2599) Remove `pytest-durations` dependency from `testing` and use native pytest option `--durations`
+- [#2597](https://github.com/meltano/sdk/issues/2597) Mark pagination classes with `@override` decorator
+- [#2596](https://github.com/meltano/sdk/issues/2596) Made `auth_headers` and `auth_params` of `APIAuthenticatorBase` simple instance attributes instead of decorated properties
+
+### 📚 Documentation Improvements
+
+- [#2639](https://github.com/meltano/sdk/issues/2639) Documented versions where `fake` and `Faker` objects were added to the stream maps context
+- [#2629](https://github.com/meltano/sdk/issues/2629) Reference `get_starting_timestamp` in incremental replication guide
+- [#2604](https://github.com/meltano/sdk/issues/2604) Update project sample links
+- [#2595](https://github.com/meltano/sdk/issues/2595) Documented examples of stream glob expressions and property aliasing
+
+### 📦 Packaging changes
+
+- [#2640](https://github.com/meltano/sdk/issues/2640) Remove upper constraint on `faker` extra
+
 ## v0.39.1 (2024-08-07)
 
 ### 🐛 Fixes

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml
@@ -3,7 +3,26 @@
 
 name: Test {{cookiecutter.mapper_id}}
 
-on: [push]
+on:
+  push:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  pull_request:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  workflow_dispatch:
 
 jobs:
   pytest:
@@ -13,7 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python {{ '${{ matrix.python-version }}' }}

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
@@ -18,19 +18,19 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -31,12 +31,12 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.39.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml
@@ -3,7 +3,26 @@
 
 name: Test {{cookiecutter.tap_id}}
 
-on: [push]
+on:
+  push:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  pull_request:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  workflow_dispatch:
 
 jobs:
   pytest:
@@ -13,7 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python {{ '${{ matrix.python-version }}' }}

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
@@ -18,20 +18,20 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
     additional_dependencies:

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -31,7 +31,7 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.8"
 importlib-resources = { version = "==6.4.*", python = "<3.9" }
-singer-sdk = { version="~=0.39.1", extras = [
+singer-sdk = { version="~=0.40.0", extras = [
     {%- if cookiecutter.auth_method == "JWT" -%}"jwt", {% endif -%}
     {%- if cookiecutter.faker_extra -%}"faker",{%- endif -%}
 ] }
@@ -43,9 +43,9 @@ requests = "~=2.32.3"
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
 {%- if cookiecutter.auth_method == "JWT" %}
-singer-sdk = { version="~=0.39.1", extras = ["jwt", "testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["jwt", "testing"] }
 {%- else %}
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["testing"] }
 {%- endif %}
 
 [tool.poetry.extras]

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py
@@ -50,6 +50,16 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
             default="https://api.mysample.com",
             description="The url for the API service",
         ),
+        {%- if cookiecutter.stream_type in ("GraphQL", "REST") %}
+        th.Property(
+            "user_agent",
+            th.StringType,
+            description=(
+                "A custom User-Agent header to send with each request. Default is "
+                "'<tap_name>/<tap_version>'"
+            ),
+        ),
+        {%- endif %}
     ).to_dict()
 {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}