From cb8676f5a6439b522d2d1b291cfcb93fd69a9828 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Sat, 11 Feb 2023 20:20:27 +0000 Subject: [PATCH 1/9] Add github action for end-to-end testing of cookiecutters --- .github/workflows/cookiecutter-e2e.yml | 68 +++++++++++++++++++ e2e-tests/cookiecutters/README.md | 17 +++++ e2e-tests/cookiecutters/tap-graphql-jwt.json | 14 ++++ .../tap-rest-api_key-github.json | 14 ++++ .../cookiecutters/target-per_record.json | 12 ++++ e2e-tests/cookiecutters/test_cookiecutter.sh | 52 ++++++++++++++ 6 files changed, 177 insertions(+) create mode 100644 .github/workflows/cookiecutter-e2e.yml create mode 100644 e2e-tests/cookiecutters/README.md create mode 100644 e2e-tests/cookiecutters/tap-graphql-jwt.json create mode 100644 e2e-tests/cookiecutters/tap-rest-api_key-github.json create mode 100644 e2e-tests/cookiecutters/target-per_record.json create mode 100644 e2e-tests/cookiecutters/test_cookiecutter.sh diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml new file mode 100644 index 000000000..8850a272f --- /dev/null +++ b/.github/workflows/cookiecutter-e2e.yml @@ -0,0 +1,68 @@ +name: E2E Cookiecutters + +on: + push: +# branches: [main] +# paths: [cookiecutter/**] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + +jobs: + lint: + name: Lint ${{ matrix.cookiecutter }}, Replay ${{ matrix.replay }} on ${{ matrix.python-version }} ${{ matrix.python-version }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + python-version: ["3.9"] + os: ["ubuntu-latest"] + include: + - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json"} + + steps: + - name: Check out the repository + uses: actions/checkout@v3.3.0 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4.5.0 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + cache: 'pip' + cache-dependency-path: 'poetry.lock' + + - name: Upgrade pip + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt + run: | + pip install pip + pip --version + + + - name: Install Poetry, Tox & Cookiecutter + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt + run: | + pipx install poetry + pipx install cookiecutter + pipx install tox + + - name: Build cookiecutter project + env: + CC_TEMPLATE: cookiecutter/${{ matrix.cookiecutter }} + REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} + run: | + bash e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 + + - name: Run lint + env: + REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} + run: | + CC_OUTPUT_DIR=$(basename $REPLAY_FILE .json) + cd /tmp/$CC_OUTPUT_DIR + poetry run tox -e lint diff --git a/e2e-tests/cookiecutters/README.md b/e2e-tests/cookiecutters/README.md new file mode 100644 index 000000000..7adc8522c --- /dev/null +++ b/e2e-tests/cookiecutters/README.md @@ -0,0 +1,17 @@ +# CI for Empty Cookiecutters + +Cookiecutters for taps and targets include two kinds of tests: linting and end-to-end testing with pytest. When a new project is created with the cookiecutter we expect: + +- linting tests should pass +- integration tests may fail (because no integration has been implemented yet) + +To automate creation of cookiecutter test projects, we use a [replay file](https://cookiecutter.readthedocs.io/en/stable/advanced/replay.html) generated by cookiecutter. + +## Running Manually + +Run a test against tap-template cookiecutter against the `tap-rest-api_key-github.json` replay file, execute: + +```bash +bash test_cookiecutter.sh ../../cookiecutter/tap-template ./tap-rest-api_key-github.json +bash test_cookiecutter.sh ../../cookiecutter/target-template ./target-per_record.json +``` diff --git a/e2e-tests/cookiecutters/tap-graphql-jwt.json b/e2e-tests/cookiecutters/tap-graphql-jwt.json new file mode 100644 index 000000000..5f9af1bcc --- /dev/null +++ b/e2e-tests/cookiecutters/tap-graphql-jwt.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "GraphQLJWTTemplateTest", + "admin_name": "Automatic Tester", + "tap_id": "tap-graphql-jwt", + "library_name": "tap_graphql_jwt", + "variant": "None (Skip)", + "stream_type": "GraphQL", + "auth_method": "JWT", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} \ No newline at end of file diff --git a/e2e-tests/cookiecutters/tap-rest-api_key-github.json b/e2e-tests/cookiecutters/tap-rest-api_key-github.json new file mode 100644 index 000000000..14963cb74 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-api_key-github.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-api_key-github", + "library_name": "tap_rest_api_key_github", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "API Key", + "include_cicd_sample_template": "GitHub", + "_template": "../tap-template/", + "_output_dir": "." + } +} \ No newline at end of file diff --git a/e2e-tests/cookiecutters/target-per_record.json b/e2e-tests/cookiecutters/target-per_record.json new file mode 100644 index 000000000..d8f1618ab --- /dev/null +++ b/e2e-tests/cookiecutters/target-per_record.json @@ -0,0 +1,12 @@ +{ + "cookiecutter": { + "destination_name": "MyDestinationName", + "admin_name": "FirstName LastName", + "target_id": "target-per_record", + "library_name": "target_per_record", + "variant": "None (Skip)", + "serialization_method": "Per record", + "_template": "./sdk/cookiecutter/target-template", + "_output_dir": "." + } + } \ No newline at end of file diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh new file mode 100644 index 000000000..d82f31be8 --- /dev/null +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -0,0 +1,52 @@ +#!/bin/bash +CC_BUILD_PATH=/tmp +TAP_TEMPLATE=$(realpath $1) +REPLAY_FILE=$(realpath $2) +CC_OUTPUT_DIR=$(basename $REPLAY_FILE .json) # name of replay file without .json +RUN_LINT=${3:-1} + +usage() { + echo "test_cookiecutter.sh [tap_template] [replay_file.json]" + echo + echo "Uses the tap template to build an empty cookiecutter, and runs the lint task on the created test project" + echo "" + if [[ $# -eq 1 ]]; then + echo "ERROR: $1" + fi +} + +if [[ ! -d $TAP_TEMPLATE ]]; then + usage "Tap template folder not found" + exit +fi +if [[ ! -f $REPLAY_FILE ]]; then + usage "Replay file not found" + exit +fi + +CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR +if [[ -d "$CC_TEST_OUTPUT" ]]; then + rm -fr "$CC_TEST_OUTPUT" +fi +#echo cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH; exit +cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH && + cd $CC_TEST_OUTPUT && + pwd && + poetry install + +if [[ $? -ne 0 ]]; then + exit $? +fi + +# before linting, auto-fix what can be autofixed +LIBRARY_NAME=$(ls * -d | egrep "tap|target") +poetry run black $LIBRARY_NAME && +poetry run isort $LIBRARY_NAME && +poetry run flake8 $LIBRARY_NAME && +poetry run pydocstyle $LIBRARY_NAME && +poetry run mypy $LIBRARY_NAME --exclude="$LIBRARY_NAME/tests" +## + +if [[ $RUN_LINT -eq 1 ]] && [[ $? -eq 0 ]]; then + poetry run tox -e lint +fi \ No newline at end of file From c7979735bfe71df019bee8be91631fc4b8a1da0d Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Sun, 12 Feb 2023 12:05:21 +0000 Subject: [PATCH 2/9] fix lint violations in tap-template --- .../{{cookiecutter.library_name}}/__init__.py | 1 + .../{{cookiecutter.library_name}}/tap.py | 1 + ...phQL' == cookiecutter.stream_type %}client.py{%endif%} | 2 +- ...REST' == cookiecutter.stream_type %}client.py{%endif%} | 8 +++++--- ...SQL' != cookiecutter.stream_type %}streams.py{%endif%} | 1 - 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py index e69de29bb..91adfcd78 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py @@ -0,0 +1 @@ +"""Tap for {{ cookiecutter.source_name }}.""" \ No newline at end of file diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py index ca508aae3..2f2cbc06b 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py @@ -22,6 +22,7 @@ # TODO: Compile a list of custom stream types here # OR rewrite discover_streams() below with your custom logic. STREAM_TYPES = [ + {{ cookiecutter.source_name }}Stream, UsersStream, GroupsStream, ] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} index 9034597a9..386481a74 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} @@ -49,7 +49,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) for record in resp_json.get(""): yield record - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: """As needed, append or transform raw data to match expected structure.""" # TODO: Delete this method if not needed. return row diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index c70be712f..d053212b4 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -2,9 +2,11 @@ import requests from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable +from typing import Any, Dict, Iterable, Optional +{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} from memoization import cached +{%- endif %} from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream @@ -54,7 +56,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return APIKeyAuthenticator.create_for_stream( self, key="x-api-key", - value=self.config.get("api_key"), + value=self.config.get("api_key") or "", location="header" ) @@ -138,7 +140,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) # TODO: Parse response body and return a set of records. yield from extract_jsonpath(self.records_jsonpath, input=response.json()) - def post_process(self, row: dict, context: Optional[dict]) -> dict: + def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: """As needed, append or transform raw data to match expected structure.""" # TODO: Delete this method if not needed. return row diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} index b6bb35daa..d085e061e 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} @@ -1,7 +1,6 @@ """Stream type classes for {{ cookiecutter.tap_id }}.""" from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable from singer_sdk import typing as th # JSON Schema typing helpers From 42bf356af4f5c54df95e438bcaa82928c5ae6dcb Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Sun, 12 Feb 2023 12:15:34 +0000 Subject: [PATCH 3/9] adding target-per_record to test matrix --- .github/workflows/cookiecutter-e2e.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 8850a272f..e8d3fd633 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -19,10 +19,9 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.9"] - os: ["ubuntu-latest"] include: - - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json"} + - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: - name: Check out the repository From 8b17b3f3b00221642e081a541cf7bf17a706856d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 12 Feb 2023 13:06:27 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- e2e-tests/cookiecutters/tap-graphql-jwt.json | 2 +- e2e-tests/cookiecutters/tap-rest-api_key-github.json | 2 +- e2e-tests/cookiecutters/target-per_record.json | 2 +- e2e-tests/cookiecutters/test_cookiecutter.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cookiecutters/tap-graphql-jwt.json b/e2e-tests/cookiecutters/tap-graphql-jwt.json index 5f9af1bcc..63357fd54 100644 --- a/e2e-tests/cookiecutters/tap-graphql-jwt.json +++ b/e2e-tests/cookiecutters/tap-graphql-jwt.json @@ -11,4 +11,4 @@ "_template": "../tap-template/", "_output_dir": "." } -} \ No newline at end of file +} diff --git a/e2e-tests/cookiecutters/tap-rest-api_key-github.json b/e2e-tests/cookiecutters/tap-rest-api_key-github.json index 14963cb74..10fc9c561 100644 --- a/e2e-tests/cookiecutters/tap-rest-api_key-github.json +++ b/e2e-tests/cookiecutters/tap-rest-api_key-github.json @@ -11,4 +11,4 @@ "_template": "../tap-template/", "_output_dir": "." } -} \ No newline at end of file +} diff --git a/e2e-tests/cookiecutters/target-per_record.json b/e2e-tests/cookiecutters/target-per_record.json index d8f1618ab..4a41a31f4 100644 --- a/e2e-tests/cookiecutters/target-per_record.json +++ b/e2e-tests/cookiecutters/target-per_record.json @@ -9,4 +9,4 @@ "_template": "./sdk/cookiecutter/target-template", "_output_dir": "." } - } \ No newline at end of file + } diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index d82f31be8..503741861 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -49,4 +49,4 @@ poetry run mypy $LIBRARY_NAME --exclude="$LIBRARY_NAME/tests" if [[ $RUN_LINT -eq 1 ]] && [[ $? -eq 0 ]]; then poetry run tox -e lint -fi \ No newline at end of file +fi From 14afcee4b4154038f4e00a3f2d1f4e5f8d8d16bd Mon Sep 17 00:00:00 2001 From: flexponsive Date: Tue, 14 Feb 2023 09:13:37 +0100 Subject: [PATCH 5/9] Apply suggestions from code review Co-authored-by: Edgar R. M. --- .github/workflows/cookiecutter-e2e.yml | 3 --- ...%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} | 2 +- e2e-tests/cookiecutters/test_cookiecutter.sh | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index e8d3fd633..e4938a801 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -32,8 +32,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - cache: 'pip' - cache-dependency-path: 'poetry.lock' - name: Upgrade pip env: @@ -42,7 +40,6 @@ jobs: pip install pip pip --version - - name: Install Poetry, Tox & Cookiecutter env: PIP_CONSTRAINT: .github/workflows/constraints.txt diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index d053212b4..17141d4c5 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -56,7 +56,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return APIKeyAuthenticator.create_for_stream( self, key="x-api-key", - value=self.config.get("api_key") or "", + value=self.config.get("api_key", ""), location="header" ) diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index 503741861..ec714ea20 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -44,7 +44,7 @@ poetry run black $LIBRARY_NAME && poetry run isort $LIBRARY_NAME && poetry run flake8 $LIBRARY_NAME && poetry run pydocstyle $LIBRARY_NAME && -poetry run mypy $LIBRARY_NAME --exclude="$LIBRARY_NAME/tests" +poetry run mypy $LIBRARY_NAME ## if [[ $RUN_LINT -eq 1 ]] && [[ $? -eq 0 ]]; then From 219e0777f5789ec67c5a483040a948ae09d2e049 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Tue, 14 Feb 2023 08:21:57 +0000 Subject: [PATCH 6/9] add missing poetry lock step; formatting --- e2e-tests/cookiecutters/test_cookiecutter.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index ec714ea20..cb7c29d38 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -28,10 +28,11 @@ CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR if [[ -d "$CC_TEST_OUTPUT" ]]; then rm -fr "$CC_TEST_OUTPUT" fi -#echo cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH; exit + cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH && cd $CC_TEST_OUTPUT && pwd && + poetry lock && poetry install if [[ $? -ne 0 ]]; then @@ -41,10 +42,10 @@ fi # before linting, auto-fix what can be autofixed LIBRARY_NAME=$(ls * -d | egrep "tap|target") poetry run black $LIBRARY_NAME && -poetry run isort $LIBRARY_NAME && -poetry run flake8 $LIBRARY_NAME && -poetry run pydocstyle $LIBRARY_NAME && -poetry run mypy $LIBRARY_NAME + poetry run isort $LIBRARY_NAME && + poetry run flake8 $LIBRARY_NAME && + poetry run pydocstyle $LIBRARY_NAME && + poetry run mypy $LIBRARY_NAME ## if [[ $RUN_LINT -eq 1 ]] && [[ $? -eq 0 ]]; then From 59bc837db84179545b9fb597eaeafd8f77d95256 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Tue, 14 Feb 2023 08:32:31 +0000 Subject: [PATCH 7/9] Finishing up Github Action with Upload Artifact --- .github/workflows/cookiecutter-e2e.yml | 10 ++++++++-- e2e-tests/cookiecutters/test_cookiecutter.sh | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index e4938a801..0b892e0a9 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -55,10 +55,16 @@ jobs: run: | bash e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.replay }} + path: | + ${{ env.CC_TEST_OUTPUT }}/ + !${{ env.CC_TEST_OUTPUT }}/.mypy_cache/ + - name: Run lint env: REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} run: | - CC_OUTPUT_DIR=$(basename $REPLAY_FILE .json) - cd /tmp/$CC_OUTPUT_DIR + cd $CC_TEST_OUTPUT poetry run tox -e lint diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index cb7c29d38..307cce49b 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -29,6 +29,10 @@ if [[ -d "$CC_TEST_OUTPUT" ]]; then rm -fr "$CC_TEST_OUTPUT" fi +if [[ -f $GITHUB_ENV ]]; then + echo CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR >> $GITHUB_ENV +fi + cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH && cd $CC_TEST_OUTPUT && pwd && From e18a77c80537ad3019315477d40260ea2afa1978 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Tue, 14 Feb 2023 08:59:40 +0000 Subject: [PATCH 8/9] run e2e github action on main branch, only when relevant files changed --- .github/workflows/cookiecutter-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 0b892e0a9..36dfc4d22 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -2,8 +2,8 @@ name: E2E Cookiecutters on: push: -# branches: [main] -# paths: [cookiecutter/**] + branches: [main] + paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From 958f7699341affb76416c8003b7c1262bed650a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 14 Feb 2023 16:43:53 -0600 Subject: [PATCH 9/9] Fix files --- .github/workflows/cookiecutter-e2e.yml | 2 + .../{{cookiecutter.tap_id}}/mypy.ini | 6 - .../{{cookiecutter.tap_id}}/pyproject.toml | 10 +- .../{{cookiecutter.tap_id}}/tox.ini | 2 +- .../{{cookiecutter.library_name}}/__init__.py | 2 +- .../{{cookiecutter.library_name}}/tap.py | 45 +++--- ...kiecutter.stream_type %}client.py{%endif%} | 48 ++++-- ...kiecutter.stream_type %}client.py{%endif%} | 6 +- ...kiecutter.stream_type %}client.py{%endif%} | 146 ++++++++++++++---- ...iecutter.stream_type %}streams.py{%endif%} | 24 +-- ...kiecutter.stream_type %}client.py{%endif%} | 2 + ...hod in ('OAuth2', 'JWT')%}auth.py{%endif%} | 25 ++- .../{{cookiecutter.target_id}}/pyproject.toml | 3 +- e2e-tests/cookiecutters/test_cookiecutter.sh | 1 - poetry.lock | 58 +------ 15 files changed, 235 insertions(+), 145 deletions(-) delete mode 100644 cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 36dfc4d22..fcc527bdb 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -4,6 +4,7 @@ on: push: branches: [main] paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -21,6 +22,7 @@ jobs: matrix: include: - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-jwt-github.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini deleted file mode 100644 index ba621de2b..000000000 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini +++ /dev/null @@ -1,6 +0,0 @@ -[mypy] -python_version = 3.9 -warn_unused_configs = True - -[mypy-backoff.*] -ignore_missing_imports = True diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index c2bf70e58..784b49ab0 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -26,12 +26,16 @@ fs-s3fs = { version = "^1.1.1", optional = true} {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} requests = "^2.28.2" {%- endif %} +{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} +cached-property = "^1" # Remove after Python 3.7 support is dropped +{%- endif %} [tool.poetry.group.dev.dependencies] pytest = "^7.2.1" flake8 = "^5.0.4" +darglint = "^1.8.1" black = "^23.1.0" -pydocstyle = "^6.3.0" +pyupgrade = "^3.3.1" mypy = "^1.0.0" isort = "^5.11.5" {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} @@ -46,6 +50,10 @@ profile = "black" multi_line_output = 3 # Vertical Hanging Indent src_paths = "{{cookiecutter.library_name}}" +[tool.mypy] +python_version = "3.9" +warn_unused_configs = true + [build-system] requires = ["poetry-core>=1.0.8"] build-backend = "poetry.core.masonry.api" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini index e542be5ac..5f20123c6 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini @@ -40,11 +40,11 @@ commands = poetry run black --check --diff {{cookiecutter.library_name}}/ poetry run isort --check {{cookiecutter.library_name}} poetry run flake8 {{cookiecutter.library_name}} - poetry run pydocstyle {{cookiecutter.library_name}} # refer to mypy.ini for specific settings poetry run mypy {{cookiecutter.library_name}} --exclude='{{cookiecutter.library_name}}/tests' [flake8] +docstring-convention = google ignore = W503 max-line-length = 88 max-complexity = 10 diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py index 91adfcd78..b5c6e813f 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py @@ -1 +1 @@ -"""Tap for {{ cookiecutter.source_name }}.""" \ No newline at end of file +"""Tap for {{ cookiecutter.source_name }}.""" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py index 2f2cbc06b..5f2832e60 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py @@ -1,31 +1,17 @@ """{{ cookiecutter.source_name }} tap class.""" -from typing import List +from __future__ import annotations -from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap, {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Stream +from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap from singer_sdk import typing as th # JSON schema typing helpers {%- if cookiecutter.stream_type == "SQL" %} + from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream {%- else %} -# TODO: Import your custom stream types here: -from {{ cookiecutter.library_name }}.streams import ( - {{ cookiecutter.source_name }}Stream, -{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} - UsersStream, - GroupsStream, -{%- endif %} -) -{%- endif %} -{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} -# TODO: Compile a list of custom stream types here -# OR rewrite discover_streams() below with your custom logic. -STREAM_TYPES = [ - {{ cookiecutter.source_name }}Stream, - UsersStream, - GroupsStream, -] +# TODO: Import your custom stream types here: +from {{ cookiecutter.library_name }} import streams {%- endif %} @@ -41,31 +27,38 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type == th.StringType, required=True, secret=True, # Flag config as protected. - description="The token to authenticate against the API service" + description="The token to authenticate against the API service", ), th.Property( "project_ids", th.ArrayType(th.StringType), required=True, - description="Project IDs to replicate" + description="Project IDs to replicate", ), th.Property( "start_date", th.DateTimeType, - description="The earliest record date to sync" + description="The earliest record date to sync", ), th.Property( "api_url", th.StringType, default="https://api.mysample.com", - description="The url for the API service" + description="The url for the API service", ), ).to_dict() {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} - def discover_streams(self) -> List[Stream]: - """Return a list of discovered streams.""" - return [stream_class(tap=self) for stream_class in STREAM_TYPES] + def discover_streams(self) -> list[streams.{{ cookiecutter.source_name }}Stream]: + """Return a list of discovered streams. + + Returns: + A list of discovered streams. + """ + return [ + streams.GroupsStream(self), + streams.UsersStream(self), + ] {%- endif %} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} index 386481a74..32c59e832 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} @@ -1,11 +1,14 @@ """GraphQL client handling, including {{ cookiecutter.source_name }}Stream base class.""" -import requests -from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable +from __future__ import annotations + +from typing import Iterable +import requests from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} + from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator {%- endif %} @@ -16,7 +19,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) # TODO: Set the API's base URL here: @property def url_base(self) -> str: - """Return the API URL root, configurable via tap settings.""" + """Return the API URL root, configurable via tap settings. + + Returns: + The base URL for all requests. + """ return self.config["api_url"] # Alternatively, use a static string for url_base: @@ -25,14 +32,22 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} @property def authenticator(self) -> {{ cookiecutter.source_name }}Authenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return {{ cookiecutter.source_name }}Authenticator.create_for_stream(self) {%- endif %} @property def http_headers(self) -> dict: - """Return the http headers needed.""" + """Return the http headers needed. + + Returns: + A dictionary of HTTP headers. + """ headers = {} if "user_agent" in self.config: headers["User-Agent"] = self.config.get("user_agent") @@ -43,13 +58,28 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return headers def parse_response(self, response: requests.Response) -> Iterable[dict]: - """Parse the response and return an iterator of result records.""" + """Parse the response and return an iterator of result records. + + Args: + response: The HTTP ``requests.Response`` object. + + Yields: + Each record from the source. + """ # TODO: Parse response body and return a set of records. resp_json = response.json() for record in resp_json.get(""): yield record - def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: - """As needed, append or transform raw data to match expected structure.""" + def post_process(self, row: dict, context: dict | None = None) -> dict | None: + """As needed, append or transform raw data to match expected structure. + + Args: + row: An individual record from the stream. + context: The stream context. + + Returns: + The updated record dictionary, or ``None`` to skip the record. + """ # TODO: Delete this method if not needed. return row diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} index 8d6b118f6..0bf718c75 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} @@ -1,8 +1,10 @@ """Custom client handling, including {{ cookiecutter.source_name }}Stream base class.""" +from __future__ import annotations + import requests from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable +from typing import Any, Iterable from singer_sdk.streams import Stream @@ -10,7 +12,7 @@ from singer_sdk.streams import Stream class {{ cookiecutter.source_name }}Stream(Stream): """Stream class for {{ cookiecutter.source_name }} streams.""" - def get_records(self, context: Optional[dict]) -> Iterable[dict]: + def get_records(self, context: dict | None) -> Iterable[dict]: """Return a generator of record-type dictionary objects. The optional `context` argument is used to identify a specific slice of the diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index 17141d4c5..52bb62a13 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -1,27 +1,46 @@ """REST client handling, including {{ cookiecutter.source_name }}Stream base class.""" -import requests -from pathlib import Path -from typing import Any, Dict, Iterable, Optional +from __future__ import annotations -{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} -from memoization import cached -{%- endif %} +{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%} +import sys +{% endif -%} +from pathlib import Path +from typing import Any, Callable, Iterable +import requests +{% if cookiecutter.auth_method == "API Key" -%} +from singer_sdk.authenticators import APIKeyAuthenticator from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream -{%- if cookiecutter.auth_method == "API Key" %} -from singer_sdk.authenticators import APIKeyAuthenticator -{%- elif cookiecutter.auth_method == "Bearer Token" %} + +{% elif cookiecutter.auth_method == "Bearer Token" -%} from singer_sdk.authenticators import BearerTokenAuthenticator -{%- elif cookiecutter.auth_method == "Basic Auth" %} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + +{% elif cookiecutter.auth_method == "Basic Auth" -%} from singer_sdk.authenticators import BasicAuthenticator -{%- elif cookiecutter.auth_method in ("OAuth2", "JWT") %} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + +{% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator -{%- endif %} +{% endif -%} + +{%- if cookiecutter.auth_method in ("OAuth2", "JWT") -%} +if sys.version_info >= (3, 8): + from functools import cached_property +else: + from cached_property import cached_property + +{% endif -%} +_Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") @@ -42,29 +61,40 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} - @property - @cached - def authenticator(self) -> {{ cookiecutter.source_name }}Authenticator: - """Return a new authenticator object.""" + @cached_property + def authenticator(self) -> _Auth: + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return {{ cookiecutter.source_name }}Authenticator.create_for_stream(self) {%- elif cookiecutter.auth_method == "API Key" %} @property def authenticator(self) -> APIKeyAuthenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return APIKeyAuthenticator.create_for_stream( self, key="x-api-key", value=self.config.get("api_key", ""), - location="header" + location="header", ) {%- elif cookiecutter.auth_method == "Bearer Token" %} @property def authenticator(self) -> BearerTokenAuthenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return BearerTokenAuthenticator.create_for_stream( self, token=self.config.get("api_key") @@ -74,7 +104,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) @property def authenticator(self) -> BasicAuthenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return BasicAuthenticator.create_for_stream( self, username=self.config.get("username"), @@ -85,7 +119,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) @property def http_headers(self) -> dict: - """Return the http headers needed.""" + """Return the http headers needed. + + Returns: + A dictionary of HTTP headers. + """ headers = {} if "user_agent" in self.config: headers["User-Agent"] = self.config.get("user_agent") @@ -96,9 +134,19 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return headers def get_next_page_token( - self, response: requests.Response, previous_token: Optional[Any] - ) -> Optional[Any]: - """Return a token for identifying next page or None if no more pages.""" + self, + response: requests.Response, + previous_token: Any | None, + ) -> Any | None: + """Return a token for identifying next page or None if no more pages. + + Args: + response: The HTTP ``requests.Response`` object. + previous_token: The previous page token value. + + Returns: + The next pagination token. + """ # TODO: If pagination is required, return a token which can be used to get the # next page. If this is the final page, return "None" to end the # pagination loop. @@ -114,9 +162,19 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return next_page_token def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] - ) -> Dict[str, Any]: - """Return a dictionary of values to be used in URL parameterization.""" + self, + context: dict | None, + next_page_token: Any | None, + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization. + + Args: + context: The stream context. + next_page_token: The next page index or value. + + Returns: + A dictionary of URL query parameters. + """ params: dict = {} if next_page_token: params["page"] = next_page_token @@ -126,21 +184,45 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return params def prepare_request_payload( - self, context: Optional[dict], next_page_token: Optional[Any] - ) -> Optional[dict]: + self, + context: dict | None, + next_page_token: Any | None, + ) -> dict | None: """Prepare the data payload for the REST API request. By default, no payload will be sent (return None). + + Args: + context: The stream context. + next_page_token: The next page index or value. + + Returns: + A dictionary with the JSON body for a POST requests. """ # TODO: Delete this method if no payload is required. (Most REST APIs.) return None def parse_response(self, response: requests.Response) -> Iterable[dict]: - """Parse the response and return an iterator of result records.""" + """Parse the response and return an iterator of result records. + + Args: + response: The HTTP ``requests.Response`` object. + + Yields: + Each record from the source. + """ # TODO: Parse response body and return a set of records. yield from extract_jsonpath(self.records_jsonpath, input=response.json()) - def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: - """As needed, append or transform raw data to match expected structure.""" + def post_process(self, row: dict, context: dict | None = None) -> dict | None: + """As needed, append or transform raw data to match expected structure. + + Args: + row: An individual record from the stream. + context: The stream context. + + Returns: + The updated record dictionary, or ``None`` to skip the record. + """ # TODO: Delete this method if not needed. return row diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} index d085e061e..5ab485498 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} @@ -1,5 +1,7 @@ """Stream type classes for {{ cookiecutter.tap_id }}.""" +from __future__ import annotations + from pathlib import Path from singer_sdk import typing as th # JSON Schema typing helpers @@ -17,6 +19,7 @@ SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") class UsersStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "users" # Optionally, you may also use `schema_filepath` in place of `schema`: # schema_filepath = SCHEMAS_DIR / "users.json" @@ -25,17 +28,17 @@ class UsersStream({{ cookiecutter.source_name }}Stream): th.Property( "id", th.StringType, - description="The user's system ID" + description="The user's system ID", ), th.Property( "age", th.IntegerType, - description="The user's age in years" + description="The user's age in years", ), th.Property( "email", th.StringType, - description="The user's email address" + description="The user's email address", ), th.Property( "address", @@ -45,10 +48,10 @@ class UsersStream({{ cookiecutter.source_name }}Stream): th.Property( "state", th.StringType, - description="State name in ISO 3166-2 format" + description="State name in ISO 3166-2 format", ), th.Property("zip", th.StringType), - ) + ), ), ).to_dict() primary_keys = ["id"] @@ -71,6 +74,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream): class GroupsStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "groups" schema = th.PropertiesList( th.Property("name", th.StringType), @@ -95,6 +99,7 @@ class GroupsStream({{ cookiecutter.source_name }}Stream): class UsersStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "users" {%- if cookiecutter.stream_type == "REST" %} path = "/users" @@ -108,24 +113,24 @@ class UsersStream({{ cookiecutter.source_name }}Stream): th.Property( "id", th.StringType, - description="The user's system ID" + description="The user's system ID", ), th.Property( "age", th.IntegerType, - description="The user's age in years" + description="The user's age in years", ), th.Property( "email", th.StringType, - description="The user's email address" + description="The user's email address", ), th.Property("street", th.StringType), th.Property("city", th.StringType), th.Property( "state", th.StringType, - description="State name in ISO 3166-2 format" + description="State name in ISO 3166-2 format", ), th.Property("zip", th.StringType), ).to_dict() @@ -133,6 +138,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream): class GroupsStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "groups" {%- if cookiecutter.stream_type == "REST" %} path = "/groups" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} index 6217b9015..f22e4dd93 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} @@ -3,6 +3,8 @@ This includes {{ cookiecutter.source_name }}Stream and {{ cookiecutter.source_name }}Connector. """ +from __future__ import annotations + import sqlalchemy from singer_sdk import SQLConnector, SQLStream diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} index f4ff1b16c..d1d87ca3d 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} @@ -1,9 +1,13 @@ """{{ cookiecutter.source_name }} Authentication.""" -{% if cookiecutter.auth_method not in ("Basic Auth", "OAuth2", "JWT") %} +from __future__ import annotations + +{%- if cookiecutter.auth_method not in ("Basic Auth", "OAuth2", "JWT") %} + # TODO: Delete this file or add custom authentication logic as needed. {%- elif cookiecutter.auth_method == "OAuth2" %} + from singer_sdk.authenticators import OAuthAuthenticator, SingletonMeta @@ -27,12 +31,21 @@ class {{ cookiecutter.source_name }}Authenticator(OAuthAuthenticator, metaclass= @classmethod def create_for_stream(cls, stream) -> "{{ cookiecutter.source_name }}Authenticator": + """Instantiate an authenticator for a specific Singer stream. + + Args: + stream: The Singer stream instance. + + Returns: + A new authenticator. + """ return cls( stream=stream, auth_endpoint="TODO: OAuth Endpoint URL", oauth_scopes="TODO: OAuth Scopes", ) {%- elif cookiecutter.auth_method == "JWT" %} + from singer_sdk.authenticators import OAuthJWTAuthenticator @@ -41,9 +54,17 @@ class {{ cookiecutter.source_name }}Authenticator(OAuthJWTAuthenticator): @classmethod def create_for_stream(cls, stream) -> "{{ cookiecutter.source_name }}Authenticator": + """Instantiate an authenticator for a specific Singer stream. + + Args: + stream: The Singer stream instance. + + Returns: + A new authenticator. + """ return cls( stream=stream, auth_endpoint="TODO: OAuth Endpoint URL", oauth_scopes="TODO: OAuth Scopes", ) -{% endif %} +{%- endif %} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 052519280..68879ac6a 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -30,8 +30,9 @@ requests = "^2.28.2" [tool.poetry.dev-dependencies] pytest = "^7.2.1" flake8 = "^5.0.4" +darglint = "^1.8.1" black = "^23.1.0" -pydocstyle = "^6.3.0" +pyupgrade = "^3.3.1" mypy = "^1.0.0" isort = "^5.11.5" {%- if cookiecutter.serialization_method != "SQL" %} diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index 307cce49b..8ab387f50 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -48,7 +48,6 @@ LIBRARY_NAME=$(ls * -d | egrep "tap|target") poetry run black $LIBRARY_NAME && poetry run isort $LIBRARY_NAME && poetry run flake8 $LIBRARY_NAME && - poetry run pydocstyle $LIBRARY_NAME && poetry run mypy $LIBRARY_NAME ## diff --git a/poetry.lock b/poetry.lock index 1f68326ec..144e5d6ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -566,8 +566,6 @@ files = [ {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, @@ -1051,7 +1049,6 @@ category = "main" optional = true python-versions = "*" files = [ - {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"}, {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, ] @@ -1632,7 +1629,7 @@ files = [ cffi = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5)", "sphinx_rtd_theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] @@ -1814,13 +1811,6 @@ files = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2256,46 +2246,6 @@ category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "SQLAlchemy-1.4.46-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:7001f16a9a8e06488c3c7154827c48455d1c1507d7228d43e781afbc8ceccf6d"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c7a46639ba058d320c9f53a81db38119a74b8a7a1884df44d09fbe807d028aaf"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27m-win32.whl", hash = "sha256:c04144a24103135ea0315d459431ac196fe96f55d3213bfd6d39d0247775c854"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27m-win_amd64.whl", hash = "sha256:7b81b1030c42b003fc10ddd17825571603117f848814a344d305262d370e7c34"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:939f9a018d2ad04036746e15d119c0428b1e557470361aa798e6e7d7f5875be0"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b7f4b6aa6e87991ec7ce0e769689a977776db6704947e562102431474799a857"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dbf17ac9a61e7a3f1c7ca47237aac93cabd7f08ad92ac5b96d6f8dea4287fc1"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7f8267682eb41a0584cf66d8a697fef64b53281d01c93a503e1344197f2e01fe"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64cb0ad8a190bc22d2112001cfecdec45baffdf41871de777239da6a28ed74b6"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-win32.whl", hash = "sha256:5f752676fc126edc1c4af0ec2e4d2adca48ddfae5de46bb40adbd3f903eb2120"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-win_amd64.whl", hash = "sha256:31de1e2c45e67a5ec1ecca6ec26aefc299dd5151e355eb5199cd9516b57340be"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d68e1762997bfebf9e5cf2a9fd0bcf9ca2fdd8136ce7b24bbd3bbfa4328f3e4a"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d112b0f3c1bc5ff70554a97344625ef621c1bfe02a73c5d97cac91f8cd7a41e"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69fac0a7054d86b997af12dc23f581cf0b25fb1c7d1fed43257dee3af32d3d6d"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-win32.whl", hash = "sha256:887865924c3d6e9a473dc82b70977395301533b3030d0f020c38fd9eba5419f2"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-win_amd64.whl", hash = "sha256:984ee13543a346324319a1fb72b698e521506f6f22dc37d7752a329e9cd00a32"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:9167d4227b56591a4cc5524f1b79ccd7ea994f36e4c648ab42ca995d28ebbb96"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d61e9ecc849d8d44d7f80894ecff4abe347136e9d926560b818f6243409f3c86"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3ec187acf85984263299a3f15c34a6c0671f83565d86d10f43ace49881a82718"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9883f5fae4fd8e3f875adc2add69f8b945625811689a6c65866a35ee9c0aea23"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-win32.whl", hash = "sha256:535377e9b10aff5a045e3d9ada8a62d02058b422c0504ebdcf07930599890eb0"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-win_amd64.whl", hash = "sha256:18cafdb27834fa03569d29f571df7115812a0e59fd6a3a03ccb0d33678ec8420"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a1ad90c97029cc3ab4ffd57443a20fac21d2ec3c89532b084b073b3feb5abff3"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4847f4b1d822754e35707db913396a29d874ee77b9c3c3ef3f04d5a9a6209618"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c5a99282848b6cae0056b85da17392a26b2d39178394fc25700bcf967e06e97a"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4b1cc7835b39835c75cf7c20c926b42e97d074147c902a9ebb7cf2c840dc4e2"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-win32.whl", hash = "sha256:c522e496f9b9b70296a7675272ec21937ccfc15da664b74b9f58d98a641ce1b6"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-win_amd64.whl", hash = "sha256:ae067ab639fa499f67ded52f5bc8e084f045d10b5ac7bb928ae4ca2b6c0429a5"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:e3c1808008124850115a3f7e793a975cfa5c8a26ceeeb9ff9cbb4485cac556df"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d164df3d83d204c69f840da30b292ac7dc54285096c6171245b8d7807185aa"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b33ffbdbbf5446cf36cd4cc530c9d9905d3c2fe56ed09e25c22c850cdb9fac92"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d94682732d1a0def5672471ba42a29ff5e21bb0aae0afa00bb10796fc1e28dd"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-win32.whl", hash = "sha256:f8cb80fe8d14307e4124f6fad64dfd87ab749c9d275f82b8b4ec84c84ecebdbe"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-win_amd64.whl", hash = "sha256:07e48cbcdda6b8bc7a59d6728bd3f5f574ffe03f2c9fb384239f3789c2d95c2e"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:1b1e5e96e2789d89f023d080bee432e2fef64d95857969e70d3cadec80bd26f0"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3714e5b33226131ac0da60d18995a102a17dddd42368b7bdd206737297823ad"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:955162ad1a931fe416eded6bb144ba891ccbf9b2e49dc7ded39274dd9c5affc5"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6e4cb5c63f705c9d546a054c60d326cbde7421421e2d2565ce3e2eee4e1a01f"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-win32.whl", hash = "sha256:51e1ba2884c6a2b8e19109dc08c71c49530006c1084156ecadfaadf5f9b8b053"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-win_amd64.whl", hash = "sha256:315676344e3558f1f80d02535f410e80ea4e8fddba31ec78fe390eff5fb8f466"}, {file = "SQLAlchemy-1.4.46.tar.gz", hash = "sha256:6913b8247d8a292ef8315162a51931e2b40ce91681f1b6f18f697045200c4a30"}, ] @@ -2305,7 +2255,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] @@ -2315,14 +2265,14 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlalchemy2-stubs"