Merge branch 'main' into refactor/cli-oop

meltano · Dec 21, 2022 · d3c414f · d3c414f
2 parents 8794cbd + 33147bd
commit d3c414f
Show file tree

Hide file tree

Showing 104 changed files with 4,646 additions and 2,554 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.12.0"
+      placeholder: "0.16.0"
     validations:
       required: true
   - type: dropdown

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -31,7 +31,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v3.2.0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,5 +1,4 @@
-pip==22.3
-poetry==1.2.2
-virtualenv==20.16.5
-nox==2022.8.7
-nox-poetry==1.0.1
+pip==22.3.1
+poetry==1.3.1
+nox==2022.11.21
+nox-poetry==1.0.2
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repository
-        uses: actions/checkout@v3.0.2
+        uses: actions/checkout@v3.2.0
 
       - name: GitHub dependency vulnerability check
         if: ${{ github.event_name == 'pull_request_target' }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -11,10 +11,10 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3.0.2
+      uses: actions/checkout@v3.2.0
 
     - name: Set up Python
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: "3.10"
 
@@ -47,7 +47,7 @@ jobs:
         file_glob: true
 
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.6.4
       with:
         user: __token__
         password: ${{ secrets.PYPI_SECRET_TOKEN }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -29,20 +29,23 @@ jobs:
         - { session: tests,   python-version: "3.8",  os: "ubuntu-latest" }
         - { session: tests,   python-version: "3.9",  os: "ubuntu-latest" }
         - { session: tests,   python-version: "3.10", os: "ubuntu-latest" }
+        - { session: tests,   python-version: "3.11", os: "ubuntu-latest" }
         - { session: tests,   python-version: "3.7",  os: "macos-latest" }
         - { session: tests,   python-version: "3.8",  os: "macos-latest" }
         - { session: tests,   python-version: "3.9",  os: "macos-latest" }
         - { session: tests,   python-version: "3.10", os: "macos-latest" }
+        - { session: tests,   python-version: "3.11", os: "macos-latest" }
         - { session: tests,   python-version: "3.7",  os: "windows-latest" }
         - { session: tests,   python-version: "3.8",  os: "windows-latest" }
         - { session: tests,   python-version: "3.9",  os: "windows-latest" }
         - { session: tests,   python-version: "3.10", os: "windows-latest" }
+        - { session: tests,   python-version: "3.11", os: "windows-latest" }
         - { session: doctest, python-version: "3.10", os: "ubuntu-latest" }
         - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest" }
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.0.2
+      uses: actions/checkout@v3.2.0
 
     - name: Install Poetry
       env:
@@ -52,7 +55,7 @@ jobs:
         poetry --version
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64
@@ -80,7 +83,7 @@ jobs:
 
     - name: Upload coverage data
       if: always() && (matrix.session == 'tests')
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         name: coverage-data
         path: ".coverage.*"
@@ -100,7 +103,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.0.2
+      uses: actions/checkout@v3.2.0
 
     - name: Install Poetry
       env:
@@ -110,7 +113,7 @@ jobs:
         poetry --version
 
     - name: Setup Python 3.10
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: '3.10'
         architecture: x64
@@ -142,15 +145,15 @@ jobs:
     needs: tests
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.0.2
+      uses: actions/checkout@v3.2.0
 
     - name: Install Poetry
       run: |
         pipx install --pip-args=--constraint=.github/workflows/constraints.txt poetry
         poetry --version
 
     - name: Set up Python
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: '3.10'
         cache: 'pip'
@@ -162,7 +165,7 @@ jobs:
         pip --version
 
     - name: Download coverage data
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         name: coverage-data
 

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -35,19 +35,23 @@ jobs:
       pull-requests: write  # to create and update PRs
 
     steps:
-    - uses: actions/checkout@v3.0.2
+    - uses: actions/checkout@v3.2.0
       with:
         fetch-depth: 0
 
+    - name: Set repo file permissions
+      run: |
+        sudo chown -R $USER:$USER .git/objects
+
     - name: Set up Python
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         python-version: "3.10"
         architecture: x64
 
     - name: Bump version
       id: cz-bump
-      uses: commitizen-tools/[email protected].0
+      uses: commitizen-tools/[email protected].1
       with:
         increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }}
         prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }}
@@ -89,4 +93,3 @@ jobs:
         base: main
         labels: release
         assignees: "${{ github.actor }}"
-        token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ ci:
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.3.0
+  rev: v4.4.0
   hooks:
   - id: check-json
   - id: check-toml
@@ -20,7 +20,13 @@ repos:
             cookiecutter/.*/meltano.yml
         )$
   - id: end-of-file-fixer
-    exclude: (cookiecutter/.*|docs/.*|samples/.*\.json)
+    exclude: |
+      (?x)^(
+        cookiecutter/.*|
+        docs/.*|
+        samples/.*\.json|
+        tests/snapshots/.*/.*\.json
+      )$
   - id: trailing-whitespace
     exclude: |
         (?x)^(
@@ -30,7 +36,7 @@ repos:
         )$
 
 - repo: https://github.com/psf/black
-  rev: 22.10.0
+  rev: 22.12.0
   hooks:
   - id: black
     exclude: |
@@ -41,13 +47,13 @@ repos:
         )$
 
 - repo: https://github.com/pycqa/isort
-  rev: 5.10.1
+  rev: v5.11.3
   hooks:
   - id: isort
     exclude: (cookiecutter/.*|singer_sdk/helpers/_simpleeval/.*)
 
 - repo: https://github.com/pycqa/flake8
-  rev: 5.0.4
+  rev: 6.0.0
   hooks:
   - id: flake8
     additional_dependencies:
@@ -57,7 +63,7 @@ repos:
     files: 'singer_sdk/.*'
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.1.0
+  rev: v3.3.1
   hooks:
   - id: pyupgrade
     args: [--py37-plus]
@@ -68,7 +74,7 @@ repos:
         )$
 
 - repo: https://github.com/python-poetry/poetry
-  rev: 1.2.2
+  rev: 1.3.1
   hooks:
   - id: poetry-check
   - id: poetry-lock

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,90 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
+## v0.16.0 (2022-12-19)
+
+### ✨ New
+
+- [#1262](https://github.com/meltano/sdk/issues/1262) Support string `"__NULL__"` whereever null values are allowed in stream maps configuration
+
+### 🐛 Fixes
+
+- [#1281](https://github.com/meltano/sdk/issues/1281) Apply Version bump commit file perms with sudo
+- [#1280](https://github.com/meltano/sdk/issues/1280) Set repo file perms after checkout in Version bump workflow
+- [#1214](https://github.com/meltano/sdk/issues/1214) Avoid duplicate entries in `required` array of JSON schema helpers
+
+## v0.15.0 (2022-12-08)
+
+### ✨ New
+
+- [#1188](https://github.com/meltano/sdk/issues/1188) Support boolean `additional_properties` in JSON schema helper objects
+- [#1237](https://github.com/meltano/sdk/issues/1237) Catch and retry `ConnectionResetError` exceptions in HTTP taps
+- [#1087](https://github.com/meltano/sdk/issues/1087) S3 batch storage -- _**Thanks @jamielxcarter!**_
+- [#1197](https://github.com/meltano/sdk/issues/1197) Support `patternProperties` in JSON schema helpers
+- [#1157](https://github.com/meltano/sdk/issues/1157) Built-in handling of `default-target-schema` for SQL Targets -- _**Thanks @BuzzCutNorman!**_
+
+### 🐛 Fixes
+
+- [#1238](https://github.com/meltano/sdk/issues/1238) Ensure metric tags coming from stream context can be JSON-serialized
+- [#1233](https://github.com/meltano/sdk/issues/1233) Add level and logger name to default log format
+- [#1219](https://github.com/meltano/sdk/issues/1219) Schema passthrough for whitelisted fields
+- [#1174](https://github.com/meltano/sdk/issues/1174) Do not emit log message if no record properties were ignored
+- [#1192](https://github.com/meltano/sdk/issues/1192) Change max record age for emitting state messages to 5 instead of 30 mins -- _**Thanks @spacecowboy!**_
+
+### ⚡ Performance Improvements
+
+- [#1196](https://github.com/meltano/sdk/issues/1196) Improve performance of record message serialization -- _**Thanks @Jack-Burnett!**_
+
+### 📚 Documentation Improvements
+
+- [#1243](https://github.com/meltano/sdk/issues/1243) Document inherited `PluginBase` attributes and methods
+- [#1209](https://github.com/meltano/sdk/issues/1209) Fix argument descriptions for `OAuthAuthenticator`
+
+## v0.14.0 (2022-11-16)
+
+### ✨ New
+
+- [#1175](https://github.com/meltano/sdk/issues/1175) Add `datetime` functions to simpleeval env in stream maps -- _**Thanks @qbatten!**_
+
+### 🐛 Fixes
+
+- [#1182](https://github.com/meltano/sdk/issues/1182) Update `SQLConnector` import for SQL target cookiecutter -- _**Thanks @radbrt!**_
+- [#1168](https://github.com/meltano/sdk/issues/1168) `SQLConnector.table_exists()` to use separate `table_name` and `schema_name` instead of fully qualified name -- _**Thanks @BuzzCutNorman!**_
+- [#1164](https://github.com/meltano/sdk/issues/1164) Write a valid final state message at the end of each stream sync -- _**Thanks @laurentS!**_
+
+### ⚙️ Under the Hood
+
+- [#1114](https://github.com/meltano/sdk/issues/1114) Make DDL overridable for column `ADD`, `ALTER`, and `RENAME` operations
+
+## v0.13.1 (2022-11-08)
+
+### 🐛 Fixes
+
+- [#1126](https://github.com/meltano/sdk/issues/1126) Resolve failure in `_increment_stream_state()` for cases when `replication_method` is `LOG_BASED`
+- [#1111](https://github.com/meltano/sdk/issues/1111) Push _MAX_RECORDS_LIMIT down into SQL
+
+### ⚙️ Under the Hood
+
+- [#1091](https://github.com/meltano/sdk/issues/1091) Move SQLConnector into a separate module, for use by both SQLStream and SQLSink
+
+### 📚 Documentation Improvements
+
+- [#1133](https://github.com/meltano/sdk/issues/1133) Fix duplicate `Known Limitations` header
+- [#1118](https://github.com/meltano/sdk/issues/1118) Document `BATCH` limitations
+
+## v0.13.0 (2022-10-24)
+
+### ✨ New
+
+- [#1098](https://github.com/meltano/sdk/issues/1098) Add JSON Schema `Property` helpers for `allowed_values` (`enum`) and `examples`
+- [#1096](https://github.com/meltano/sdk/issues/1096) Add secrets support for tap and target config, via `Property(..., secret=True)`
+- [#1039](https://github.com/meltano/sdk/issues/1039) Support conforming singer property names to target identifier constraints in SQL sinks
+
+### 🐛 Fixes
+
+- [#1093](https://github.com/meltano/sdk/issues/1093) Add environment support to the cookie cutter for `meltano.yml`
+- [#1036](https://github.com/meltano/sdk/issues/1036) Create schema and table on `add_sink`
+
 ## v0.12.0 (2022-10-17)
 
 ### ✨ New

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/meltano.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/meltano.yml
@@ -1,6 +1,9 @@
 version: 1
 send_anonymous_usage_stats: true
 project_id: "{{cookiecutter.tap_id}}"
+default_environment: test
+environments:
+- name: test
 plugins:
   extractors:
   - name: "{{cookiecutter.tap_id}}"
@@ -10,6 +13,8 @@ plugins:
     - state
     - catalog
     - discover
+    - about
+    - stream-maps
     config:
       start_date: '2010-01-01T00:00:00Z'
     settings:

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -12,7 +12,8 @@ license = "Apache 2.0"
 [tool.poetry.dependencies]
 python = "<3.11,>=3.7.1"
 requests = "^2.25.1"
-singer-sdk = "^0.12.0"
+singer-sdk = { version="^0.16.0"}
+fs-s3fs = { version = "^1.1.1", optional = true}
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"
@@ -24,6 +25,9 @@ mypy = "^0.910"
 types-requests = "^2.26.1"
 isort = "^5.10.1"
 
+[tool.poetry.extras]
+s3 = ["fs-s3fs"]
+
 [tool.isort]
 profile = "black"
 multi_line_output = 3 # Vertical Hanging Indent

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py
@@ -38,6 +38,7 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
             "auth_token",
             th.StringType,
             required=True,
+            secret=True,  # Flag config as protected.
             description="The token to authenticate against the API service"
         ),
         th.Property(