From eb2b908a9d3e23d8af80e321a44bcadb1de60fbb Mon Sep 17 00:00:00 2001 From: flexponsive Date: Fri, 17 Feb 2023 00:44:37 +0100 Subject: [PATCH] fix(templates): E2E Cookiecutter - Cover all REST authentication cases + one GraphQL case (#1428) --- .github/workflows/cookiecutter-e2e.yml | 13 +++++++++++-- ...= cookiecutter.stream_type %}client.py{%endif%} | 10 +++++++--- ...h_method in ('OAuth2', 'JWT')%}auth.py{%endif%} | 6 +++++- e2e-tests/cookiecutters/tap-rest-basic_auth.json | 14 ++++++++++++++ e2e-tests/cookiecutters/tap-rest-bearer_token.json | 14 ++++++++++++++ e2e-tests/cookiecutters/tap-rest-custom.json | 14 ++++++++++++++ e2e-tests/cookiecutters/tap-rest-jwt.json | 14 ++++++++++++++ e2e-tests/cookiecutters/tap-rest-oauth2.json | 14 ++++++++++++++ e2e-tests/cookiecutters/test_cookiecutter.sh | 4 ++-- 9 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 e2e-tests/cookiecutters/tap-rest-basic_auth.json create mode 100644 e2e-tests/cookiecutters/tap-rest-bearer_token.json create mode 100644 e2e-tests/cookiecutters/tap-rest-custom.json create mode 100644 e2e-tests/cookiecutters/tap-rest-jwt.json create mode 100644 e2e-tests/cookiecutters/tap-rest-oauth2.json mode change 100644 => 100755 e2e-tests/cookiecutters/test_cookiecutter.sh diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index fcc527bdb..7013d3951 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -1,6 +1,9 @@ name: E2E Cookiecutters on: + pull_request: + types: [opened, synchronize, reopened] + paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] push: branches: [main] paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] @@ -21,8 +24,13 @@ jobs: fail-fast: true matrix: include: + - { cookiecutter: "tap-template", replay: "tap-graphql-jwt.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-jwt-github.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-basic_auth.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-bearer_token.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-custom.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-jwt.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-oauth2.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: @@ -55,7 +63,8 @@ jobs: CC_TEMPLATE: cookiecutter/${{ matrix.cookiecutter }} REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} run: | - bash e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 + chmod +x ./e2e-tests/cookiecutters/test_cookiecutter.sh + ./e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 - uses: actions/upload-artifact@v3 with: diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index 52bb62a13..dda47c913 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -24,6 +24,10 @@ from singer_sdk.authenticators import BasicAuthenticator from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream +{% elif cookiecutter.auth_method == "Custom or N/A" -%} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + {% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream @@ -97,7 +101,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) """ return BearerTokenAuthenticator.create_for_stream( self, - token=self.config.get("api_key") + token=self.config.get("api_key", ""), ) {%- elif cookiecutter.auth_method == "Basic Auth" %} @@ -111,8 +115,8 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) """ return BasicAuthenticator.create_for_stream( self, - username=self.config.get("username"), - password=self.config.get("password"), + username=self.config.get("username", ""), + password=self.config.get("password", ""), ) {%- endif %} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} index d1d87ca3d..5258a5f1f 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} @@ -18,7 +18,11 @@ class {{ cookiecutter.source_name }}Authenticator(OAuthAuthenticator, metaclass= @property def oauth_request_body(self) -> dict: - """Define the OAuth request body for the {{ cookiecutter.source_name }} API.""" + """Define the OAuth request body for the AutomaticTestTap API. + + Returns: + A dict with the request body + """ # TODO: Define the request body needed for the API. return { 'resource': 'https://analysis.windows.net/powerbi/api', diff --git a/e2e-tests/cookiecutters/tap-rest-basic_auth.json b/e2e-tests/cookiecutters/tap-rest-basic_auth.json new file mode 100644 index 000000000..baaa7caa8 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-basic_auth.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-basic_auth", + "library_name": "tap_rest_basic_auth", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Basic Auth", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} diff --git a/e2e-tests/cookiecutters/tap-rest-bearer_token.json b/e2e-tests/cookiecutters/tap-rest-bearer_token.json new file mode 100644 index 000000000..5d5e3ec5c --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-bearer_token.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-bearer_token", + "library_name": "tap_rest_bearer_token", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Bearer Token", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} diff --git a/e2e-tests/cookiecutters/tap-rest-custom.json b/e2e-tests/cookiecutters/tap-rest-custom.json new file mode 100644 index 000000000..639351bea --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-custom.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-custom", + "library_name": "tap_rest_custom", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Custom or N/A", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} diff --git a/e2e-tests/cookiecutters/tap-rest-jwt.json b/e2e-tests/cookiecutters/tap-rest-jwt.json new file mode 100644 index 000000000..789775fb6 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-jwt.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-jwt", + "library_name": "tap_rest_jwt", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "JWT", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} diff --git a/e2e-tests/cookiecutters/tap-rest-oauth2.json b/e2e-tests/cookiecutters/tap-rest-oauth2.json new file mode 100644 index 000000000..df4c11ac2 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-oauth2.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-oauth2", + "library_name": "tap_rest_oauth2", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "OAuth2", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh old mode 100644 new mode 100755 index 8ab387f50..3d5b504ed --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -17,11 +17,11 @@ usage() { if [[ ! -d $TAP_TEMPLATE ]]; then usage "Tap template folder not found" - exit + exit 1 fi if [[ ! -f $REPLAY_FILE ]]; then usage "Replay file not found" - exit + exit 1 fi CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR