From 5cf77b3e3e42ce5bff383bbef8d4ee210476cc6c Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Wed, 15 Feb 2023 11:14:36 +0000 Subject: [PATCH 1/8] fix file name in action; add error exit codes in shell script --- .github/workflows/cookiecutter-e2e.yml | 2 +- e2e-tests/cookiecutters/test_cookiecutter.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index fcc527bdb..6c5b0d4fc 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -22,7 +22,7 @@ jobs: matrix: include: - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-jwt-github.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-graphql-jwt.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index 8ab387f50..3d5b504ed 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -17,11 +17,11 @@ usage() { if [[ ! -d $TAP_TEMPLATE ]]; then usage "Tap template folder not found" - exit + exit 1 fi if [[ ! -f $REPLAY_FILE ]]; then usage "Replay file not found" - exit + exit 1 fi CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR From 8c543f59c427c90ebb09999d631893ce9b26f49f Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 12:40:21 +0000 Subject: [PATCH 2/8] add test case for tap / rest / custom auth; fix mypy error (https://github.com/meltano/sdk/commit/faf95bd494a42fa026d561668705e2b54e03d0a2#commitcomment-100709368) --- ...= cookiecutter.stream_type %}client.py{%endif%} | 4 ++++ e2e-tests/cookiecutters/tap-rest-custom.json | 14 ++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 e2e-tests/cookiecutters/tap-rest-custom.json diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index 52bb62a13..d983b2b3b 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -24,6 +24,10 @@ from singer_sdk.authenticators import BasicAuthenticator from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream +{% elif cookiecutter.auth_method == "Custom or N/A" -%} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + {% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream diff --git a/e2e-tests/cookiecutters/tap-rest-custom.json b/e2e-tests/cookiecutters/tap-rest-custom.json new file mode 100644 index 000000000..639351bea --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-custom.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-custom", + "library_name": "tap_rest_custom", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Custom or N/A", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} From 55d5b25ed6af1a9eeccb15f31be7e8ce427d21d4 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 12:43:55 +0000 Subject: [PATCH 3/8] add test case for tap / rest / bearer token; fix mypy error --- ...= cookiecutter.stream_type %}client.py{%endif%} | 2 +- e2e-tests/cookiecutters/tap-rest-bearer_token.json | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 e2e-tests/cookiecutters/tap-rest-bearer_token.json diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index d983b2b3b..5fac0acf4 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -101,7 +101,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) """ return BearerTokenAuthenticator.create_for_stream( self, - token=self.config.get("api_key") + token=self.config.get("api_key", ""), ) {%- elif cookiecutter.auth_method == "Basic Auth" %} diff --git a/e2e-tests/cookiecutters/tap-rest-bearer_token.json b/e2e-tests/cookiecutters/tap-rest-bearer_token.json new file mode 100644 index 000000000..5d5e3ec5c --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-bearer_token.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-bearer_token", + "library_name": "tap_rest_bearer_token", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Bearer Token", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} From 2cb12be4d5bc22f6c3be90e3efaf0d6c9aa1ca6d Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 12:47:39 +0000 Subject: [PATCH 4/8] adding test case for tap / rest / basic auth; fix mypy error --- ...= cookiecutter.stream_type %}client.py{%endif%} | 4 ++-- e2e-tests/cookiecutters/tap-rest-basic_auth.json | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 e2e-tests/cookiecutters/tap-rest-basic_auth.json diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index 5fac0acf4..dda47c913 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -115,8 +115,8 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) """ return BasicAuthenticator.create_for_stream( self, - username=self.config.get("username"), - password=self.config.get("password"), + username=self.config.get("username", ""), + password=self.config.get("password", ""), ) {%- endif %} diff --git a/e2e-tests/cookiecutters/tap-rest-basic_auth.json b/e2e-tests/cookiecutters/tap-rest-basic_auth.json new file mode 100644 index 000000000..baaa7caa8 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-basic_auth.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-basic_auth", + "library_name": "tap_rest_basic_auth", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Basic Auth", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} From a14d6b3ddfc77f57182e1cd64bc251e5b105a741 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 12:53:35 +0000 Subject: [PATCH 5/8] adding test case for tap / rest / oauth2; fix docstring --- ...h_method in ('OAuth2', 'JWT')%}auth.py{%endif%} | 6 +++++- e2e-tests/cookiecutters/tap-rest-oauth2.json | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 e2e-tests/cookiecutters/tap-rest-oauth2.json diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} index d1d87ca3d..5258a5f1f 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} @@ -18,7 +18,11 @@ class {{ cookiecutter.source_name }}Authenticator(OAuthAuthenticator, metaclass= @property def oauth_request_body(self) -> dict: - """Define the OAuth request body for the {{ cookiecutter.source_name }} API.""" + """Define the OAuth request body for the AutomaticTestTap API. + + Returns: + A dict with the request body + """ # TODO: Define the request body needed for the API. return { 'resource': 'https://analysis.windows.net/powerbi/api', diff --git a/e2e-tests/cookiecutters/tap-rest-oauth2.json b/e2e-tests/cookiecutters/tap-rest-oauth2.json new file mode 100644 index 000000000..df4c11ac2 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-oauth2.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-oauth2", + "library_name": "tap_rest_oauth2", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "OAuth2", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} From 51d624c4deb08a31fdb880018db53b064b22bfc3 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 12:58:24 +0000 Subject: [PATCH 6/8] adding tap / rest / jwt test case --- e2e-tests/cookiecutters/tap-rest-jwt.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 e2e-tests/cookiecutters/tap-rest-jwt.json diff --git a/e2e-tests/cookiecutters/tap-rest-jwt.json b/e2e-tests/cookiecutters/tap-rest-jwt.json new file mode 100644 index 000000000..789775fb6 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-rest-jwt.json @@ -0,0 +1,14 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "tap_id": "tap-rest-jwt", + "library_name": "tap_rest_jwt", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "JWT", + "include_cicd_sample_template": "None (Skip)", + "_template": "../tap-template/", + "_output_dir": "." + } +} From 72ab84ce628d89f23132efa3aea7d98a27667f75 Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 13:56:44 +0000 Subject: [PATCH 7/8] update test matrix in github action --- .github/workflows/cookiecutter-e2e.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 6c5b0d4fc..508e6cc9f 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -1,6 +1,9 @@ name: E2E Cookiecutters on: + pull_request: + types: [opened, synchronize, reopened] + paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] push: branches: [main] paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] @@ -21,9 +24,14 @@ jobs: fail-fast: true matrix: include: - - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "tap-template", replay: "tap-graphql-jwt.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-basic_auth.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-bearer_token.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-custom.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-jwt.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-oauth2.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: - name: Check out the repository From dd1a260aeec757c278ee4e379c5ed1054c9fe22d Mon Sep 17 00:00:00 2001 From: basteln3rk Date: Thu, 16 Feb 2023 14:25:58 +0000 Subject: [PATCH 8/8] fix typo; improve error handling in github action --- .github/workflows/cookiecutter-e2e.yml | 5 +++-- e2e-tests/cookiecutters/test_cookiecutter.sh | 0 2 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 e2e-tests/cookiecutters/test_cookiecutter.sh diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 508e6cc9f..7013d3951 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -31,7 +31,7 @@ jobs: - { cookiecutter: "tap-template", replay: "tap-rest-custom.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "tap-template", replay: "tap-rest-jwt.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "tap-template", replay: "tap-rest-oauth2.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: - name: Check out the repository @@ -63,7 +63,8 @@ jobs: CC_TEMPLATE: cookiecutter/${{ matrix.cookiecutter }} REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} run: | - bash e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 + chmod +x ./e2e-tests/cookiecutters/test_cookiecutter.sh + ./e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 - uses: actions/upload-artifact@v3 with: diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh old mode 100644 new mode 100755