feat: Build empty cookiecutters and run lint task during CI

.github/workflows/cookiecutter-e2e.yml

@@ -0,0 +1,67 @@
+name: E2E Cookiecutters
+
+on:
+  push:
+#    branches: [main]
+#    paths: [cookiecutter/**]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1"
+
+jobs:
+  lint:
+    name: Lint ${{ matrix.cookiecutter }}, Replay ${{ matrix.replay }} on ${{ matrix.python-version }} ${{ matrix.python-version }} / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9",  os: "ubuntu-latest" }
+          - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9",  os: "ubuntu-latest" }
+
+    steps:
+    - name: Check out the repository
+      uses: actions/[email protected]
+
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+        cache: 'pip'
+        cache-dependency-path: 'poetry.lock'
+
+    - name: Upgrade pip
+      env:
+        PIP_CONSTRAINT: .github/workflows/constraints.txt
+      run: |
+        pip install pip
+        pip --version
+
+
+    - name: Install Poetry, Tox & Cookiecutter
+      env:
+        PIP_CONSTRAINT: .github/workflows/constraints.txt
+      run: |
+        pipx install poetry
+        pipx install cookiecutter
+        pipx install tox
+
+    - name: Build cookiecutter project
+      env:
+        CC_TEMPLATE: cookiecutter/${{ matrix.cookiecutter }}
+        REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }}
+      run: |
+        bash e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0
+
+    - name: Run lint
+      env:
+        REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }}
+      run: |
+        CC_OUTPUT_DIR=$(basename $REPLAY_FILE .json)
+        cd /tmp/$CC_OUTPUT_DIR
+        poetry run tox -e lint

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py

@@ -0,0 +1 @@
+"""Tap for {{ cookiecutter.source_name }}."""

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py

@@ -22,6 +22,7 @@
 # TODO: Compile a list of custom stream types here
 #       OR rewrite discover_streams() below with your custom logic.
 STREAM_TYPES = [
+    {{ cookiecutter.source_name }}Stream,
     UsersStream,
     GroupsStream,
 ]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%}

@@ -49,7 +49,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
         for record in resp_json.get("<TODO>"):
             yield record
 
-    def post_process(self, row: dict, context: Optional[dict] = None) -> dict:
+    def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
         """As needed, append or transform raw data to match expected structure."""
         # TODO: Delete this method if not needed.
         return row

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%}

@@ -2,9 +2,11 @@
 
 import requests
 from pathlib import Path
-from typing import Any, Dict, Optional, Union, List, Iterable
+from typing import Any, Dict, Iterable, Optional
 
+{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
 from memoization import cached
+{%- endif %}
 
 from singer_sdk.helpers.jsonpath import extract_jsonpath
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
@@ -54,7 +56,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
         return APIKeyAuthenticator.create_for_stream(
             self,
             key="x-api-key",
-            value=self.config.get("api_key"),
+            value=self.config.get("api_key") or "",
             location="header"
         )
 
@@ -138,7 +140,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
         # TODO: Parse response body and return a set of records.
         yield from extract_jsonpath(self.records_jsonpath, input=response.json())
 
-    def post_process(self, row: dict, context: Optional[dict]) -> dict:
+    def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
         """As needed, append or transform raw data to match expected structure."""
         # TODO: Delete this method if not needed.
         return row

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%}

@@ -1,7 +1,6 @@
 """Stream type classes for {{ cookiecutter.tap_id }}."""
 
 from pathlib import Path
-from typing import Any, Dict, Optional, Union, List, Iterable
 
 from singer_sdk import typing as th  # JSON Schema typing helpers
 

e2e-tests/cookiecutters/README.md

@@ -0,0 +1,17 @@
+# CI for Empty Cookiecutters
+
+Cookiecutters for taps and targets include two kinds of tests: linting and end-to-end testing with pytest. When a new project is created with the cookiecutter we expect:
+
+- linting tests should pass
+- integration tests may fail (because no integration has been implemented yet)
+
+To automate creation of cookiecutter test projects, we use a [replay file](https://cookiecutter.readthedocs.io/en/stable/advanced/replay.html) generated by cookiecutter.
+
+## Running Manually
+
+Run a test against tap-template cookiecutter against the `tap-rest-api_key-github.json` replay file, execute:
+
+```bash
+bash test_cookiecutter.sh ../../cookiecutter/tap-template ./tap-rest-api_key-github.json
+bash test_cookiecutter.sh ../../cookiecutter/target-template ./target-per_record.json
+```

e2e-tests/cookiecutters/tap-graphql-jwt.json

@@ -0,0 +1,14 @@
+{
+    "cookiecutter": {
+      "source_name": "GraphQLJWTTemplateTest",
+      "admin_name": "Automatic Tester",
+      "tap_id": "tap-graphql-jwt",
+      "library_name": "tap_graphql_jwt",
+      "variant": "None (Skip)",
+      "stream_type": "GraphQL",
+      "auth_method": "JWT",
+      "include_cicd_sample_template": "None (Skip)",
+      "_template": "../tap-template/",
+      "_output_dir": "."
+    }
+}

e2e-tests/cookiecutters/tap-rest-api_key-github.json

@@ -0,0 +1,14 @@
+{
+    "cookiecutter": {
+      "source_name": "AutomaticTestTap",
+      "admin_name": "Automatic Tester",
+      "tap_id": "tap-rest-api_key-github",
+      "library_name": "tap_rest_api_key_github",
+      "variant": "None (Skip)",
+      "stream_type": "REST",
+      "auth_method": "API Key",
+      "include_cicd_sample_template": "GitHub",
+      "_template": "../tap-template/",
+      "_output_dir": "."
+    }
+}

e2e-tests/cookiecutters/target-per_record.json

@@ -0,0 +1,12 @@
+{
+    "cookiecutter": {
+      "destination_name": "MyDestinationName",
+      "admin_name": "FirstName LastName",
+      "target_id": "target-per_record",
+      "library_name": "target_per_record",
+      "variant": "None (Skip)",
+      "serialization_method": "Per record",
+      "_template": "./sdk/cookiecutter/target-template",
+      "_output_dir": "."
+    }
+  }

e2e-tests/cookiecutters/test_cookiecutter.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+CC_BUILD_PATH=/tmp
+TAP_TEMPLATE=$(realpath $1)
+REPLAY_FILE=$(realpath $2)
+CC_OUTPUT_DIR=$(basename $REPLAY_FILE .json) # name of replay file without .json
+RUN_LINT=${3:-1}
+
+usage() {
+    echo "test_cookiecutter.sh [tap_template] [replay_file.json]"
+    echo
+    echo "Uses the tap template to build an empty cookiecutter, and runs the lint task on the created test project"
+    echo ""
+    if [[ $# -eq 1 ]]; then
+        echo "ERROR: $1"
+    fi
+}
+
+if [[ ! -d $TAP_TEMPLATE ]]; then
+    usage "Tap template folder not found"
+    exit
+fi
+if [[ ! -f $REPLAY_FILE ]]; then
+    usage "Replay file not found"
+    exit
+fi
+
+CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR
+if [[ -d "$CC_TEST_OUTPUT" ]]; then
+    rm -fr "$CC_TEST_OUTPUT"
+fi
+#echo cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH; exit
+cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH &&
+    cd $CC_TEST_OUTPUT &&
+    pwd &&
+    poetry install
+
+if [[ $? -ne 0 ]]; then
+    exit $?
+fi
+
+# before linting, auto-fix what can be autofixed
+LIBRARY_NAME=$(ls * -d | egrep "tap|target")
+poetry run black $LIBRARY_NAME &&
+poetry run isort $LIBRARY_NAME &&
+poetry run flake8 $LIBRARY_NAME &&
+poetry run pydocstyle $LIBRARY_NAME &&
+poetry run mypy $LIBRARY_NAME --exclude="$LIBRARY_NAME/tests"
+##
+
+if [[ $RUN_LINT -eq 1 ]] && [[ $? -eq 0 ]]; then
+    poetry run tox -e lint
+fi