From e98011df11c79070573263340c04a1b75a478c98 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Sat, 30 Apr 2022 15:50:20 +0200 Subject: [PATCH 01/14] added end date configuration. --- .../advertiser_ids.yml | 140 ++++++++++++++++++ .../source_tiktok_marketing/source.py | 2 + .../source_tiktok_marketing/spec.py | 14 +- .../source_tiktok_marketing/streams.py | 15 +- 4 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml new file mode 100644 index 000000000000..1dfdff184e50 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml @@ -0,0 +1,140 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.27.1 + method: GET + uri: https://business-api.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 + response: + body: + string: 'Temporary + Redirect. + + + ' + headers: + Cache-Control: + - max-age=0, no-cache, no-store + Connection: + - keep-alive + Content-Length: + - '232' + Content-Type: + - text/html; charset=utf-8 + Date: + - Sat, 30 Apr 2022 13:41:16 GMT + Expires: + - Sat, 30 Apr 2022 13:41:16 GMT + Location: + - https://ads.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 + Pragma: + - no-cache + Server: + - nginx + Server-Timing: + - cdn-cache; desc=MISS, edge; dur=84, origin; dur=10 + - inner; dur=2 + X-Akamai-Request-ID: + - 6c168ba1.3225038f + X-Cache: + - TCP_MISS from a88-221-221-133.deploy.akamaitechnologies.com (AkamaiGHost/10.7.5-41022941) + (-) + X-Cache-Remote: + - TCP_MISS from a172-232-7-15.deploy.akamaitechnologies.com (AkamaiGHost/10.7.5-41022941) + (-) + X-Origin-Response-Time: + - 10,172.232.7.15 + X-Parent-Response-Time: + - 94,88.221.221.133 + x-tt-logid: + - 202204301341160101131350911272485E + x-tt-trace-host: + - 0187fa220474ed9f3ce919185a4a99d3a80f27baa2153625a326437072514ab1e1ea27c3af023794a41bdb590603bf0ecef0aeb1b499ac24edb430501660a9a5f4fe2361247cd1065faf1e0e95223f5845cf8c378f5f9bc6a480d2486fef9df47edc8a7040d2204554f97fec059f253a7f + x-tt-trace-tag: + - id=16;cdn-cache=miss;type=dyn + status: + code: 307 + message: Temporary Redirect +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.27.1 + method: GET + uri: https://ads.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//rJZdb+I4FIb/isXFalaaVsff9txRoKVDaStot9LurFaGGIiApGMnsyNV + /e97EqYZLvD2ZiQIsZ34Oef49Wteessy871PBD6S3t7H6NZNq3c36WFH8F9rH6t/8qzpY8AYCA6U + C0oVUGBCMkHxBgacciOgeSdzlcOnX3q7PFZ489dLz2XffKjy6MNhJmUkcMlBaGq5NsB4Qz96qnD7 + Noqxi8+OnJFxf9pc3X5Rh7UPZP7swtbF6En/Cge0MsCt6b1+JKdgSkiKEAmGSobR89Owz265JU/l + bhW3eYGzXgRXZB38c38wIU93N5fzyfUt6dcxfKkBVstY1cWarA6NQIbB1RhX8z6iQGCdUmEZaZsv + ALVYT6vs6bAud25ZFnkT0GiG17f21X4xxqbglhnJk8lbbrQQCuutpTRMJ5Jv4s9UM/90OOuSnl4/ + PIxuhqPHh/lgjPTZ4+3w8vF2gkOoAC0ZZQmupfgRRkmluOJWygR3/rV2wZNRkX/vqMd9P7LUAIYl + WZgi6tLiL2ZKUYqnWdO724dRR/mzrCoyH5HfyKA8J5NGR/iykMImOYIZBtZqygCBMsUZliGvytiR + 7v1zzAclGfq6isvNrpFVm9eXupEjEZJrzY36Hyy3oI1SRivMTp/GzsrlNlYuvM89exeJUgGrqBKU + MW20kaeRLWHqvpNfwVQcrQW4BpSOACkSzNVB/6lZDMrBSsqU1VYbY+jpWQaX913MzX272SOZH/Zx + uxtW/6IKiw32+108JHGkFcok0zoVhRUCXU0BaEbRHlNR9BeLRoW40frXeP3RfKsXCIMGSZMMQxlK + 0TAULXDBEvW6Kdd55ZebhvI4wGvX8cZRmpqEf2jgxgImCtqCNdYw9Z5/DF0koxAr39W3PxuS+1Cu + g9vvszz4bZWXBZkeSl0sN61bYj0lJOqJUWC2SkqqAf2EoWEmdrjbVR47cJmKyhcVGdeLn2b2NjjO + 0eOz3BexIv78j3Mc48AU5ihTfCOZEkJzQD7gfkx4dfQLvMu6BZ1j+3njwt4d6aeVj8Q9hU6SBirQ + Slsm0UM1GmkCeDLhD1co5ur3X5K4pcyq5qRi6OToCQnv+bn8T0fHxxP+d8gORoD0WV1kq7rYkkm7 + 8mrXLjseX2juCcs98I3F00RrptGQ2Hv82cVFx++IFz7s8uKs3eS+WOCfCBzVzdwcj+e/X1//AwAA + //8DAAZBQqwHCQAA + headers: + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Length: + - '924' + Content-Type: + - application/json + Date: + - Sat, 30 Apr 2022 13:41:16 GMT + Server: + - nginx + Server-Timing: + - cdn-cache; desc=MISS, edge; dur=173, origin; dur=204 + - inner; dur=202 + Vary: + - Accept-Encoding + X-Domain-Change-Info: + - We have moved to a new domain for API services after the Platform Upgrade + on 09/30/2021. Please update all of your requests with the new domain if you + haven't done so already. For more information, please visit https://ads.tiktok.com/marketing_api/docs?rid=esb88xalm6m&id=1709207085043713 + X-Origin-Response-Time: + - 204,23.44.4.15 + X-Parent-Response-Time: + - 372,88.221.221.133 + X-TT-LOGID: + - 202204301341160102452410100C313840 + - 202204301341160102452410100C313840 + X_TT_LOGID: + - 202204301341160102452410100C313840 + stablecode: + - '0' + upstream-caught: + - None + x-tt-trace-host: + - 0187fa220474ed9f3ce919185a4a99d3a80f27baa2153625a326437072514ab1e1843afc31012de609bffa64a8a8247d7427ab22037339612b2c7d5b2c73861bbb62cec17bafa555777465bd3758eea26daacab5ba904e00a94f47bddb6c63c2ef37b36f4b8bdb136e69e32956a8584f8b + x-tt-trace-tag: + - id=16;cdn-cache=miss;type=dyn + status: + code: 200 + message: OK +version: 1 diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index f119cf44d171..d93425f65dc8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -19,6 +19,7 @@ ) from .streams import ( DEFAULT_START_DATE, + DEFAULT_END_DATE, AdGroupAudienceReports, AdGroups, AdGroupsReports, @@ -92,6 +93,7 @@ def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]: return { "authenticator": TiktokTokenAuthenticator(access_token), "start_date": config.get("start_date") or DEFAULT_START_DATE, + "end_date": config.get("end_date") or DEFAULT_END_DATE, "advertiser_id": advertiser_id, "app_id": app_id, "secret": secret, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py index f33e829befff..f89a8d976fb0 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py @@ -10,7 +10,7 @@ from jsonschema import RefResolver from pydantic import BaseModel, Field -from .streams import DEFAULT_START_DATE, ReportGranularity +from .streams import DEFAULT_START_DATE, DEFAULT_END_DATE, ReportGranularity class OauthCredSpec(BaseModel): @@ -79,6 +79,18 @@ class Config: order=2, ) + end_date: str = Field( + title="End Date", + default=DEFAULT_END_DATE, + pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + description=( + "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. " + "All data generated between start_date and this date will be replicated. " + "Not setting this option will result in always syncing the data till the current date." + ), + order=3, + ) + @classmethod def change_format_to_oneOf(cls, schema: dict) -> dict: new_schema = {} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index d9c6fa14fef8..a2f8cae2930f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -22,6 +22,7 @@ # TikTok Initial release date is September 2016 DEFAULT_START_DATE = "2016-09-01" +DEFAULT_END_DATE = str(datetime.now().date()) NOT_AUDIENCE_METRICS = [ "reach", "cost_per_1000_reached", @@ -226,12 +227,15 @@ def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any return Decimal(original_value) return original_value - def __init__(self, start_date: str, **kwargs): + def __init__(self, start_date: str, end_date: str, **kwargs): super().__init__(**kwargs) self.kwargs = kwargs # convert a start date to TikTok format # example: "2021-08-24" => "2021-08-24 00:00:00" self._start_time = pendulum.parse(start_date or DEFAULT_START_DATE).strftime("%Y-%m-%d 00:00:00") + # convert end date to TikTok format + # example: "2021-08-24" => "2021-08-24 00:00:00" + self._end_time = pendulum.parse(end_date or DEFAULT_END_DATE).strftime("%Y-%m-%d 00:00:00") self.max_cursor_date = None self._advertiser_ids = [] @@ -422,7 +426,9 @@ def cursor_field(self): return [] @staticmethod - def _get_time_interval(start_date: Union[datetime, str], granularity: ReportGranularity) -> Iterable[Tuple[datetime, datetime]]: + def _get_time_interval(start_date: Union[datetime, str], + ending_date: Union[datetime, str], + granularity: ReportGranularity) -> Iterable[Tuple[datetime, datetime]]: """Due to time range restrictions based on the level of granularity of reports, we have to chunk API calls in order to get the desired time range. Docs: https://ads.tiktok.com/marketing_api/docs?id=1714590313280513 @@ -432,7 +438,7 @@ def _get_time_interval(start_date: Union[datetime, str], granularity: ReportGran """ if isinstance(start_date, str): start_date = pendulum.parse(start_date) - end_date = pendulum.now() + end_date = pendulum.parse(ending_date) if ending_date else pendulum.now() # Snapchat API only allows certain amount of days of data based on the reporting granularity if granularity == ReportGranularity.DAY: @@ -530,9 +536,10 @@ def _get_metrics(self): def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: stream_start = self.select_cursor_field_value(stream_state) or self._start_time + stream_end = self._end_time for slice_adv_id in super().stream_slices(**kwargs): - for start_date, end_date in self._get_time_interval(stream_start, self.report_granularity): + for start_date, end_date in self._get_time_interval(stream_start, stream_end, self.report_granularity): slice = { "advertiser_id": slice_adv_id["advertiser_id"], "start_date": start_date.strftime("%Y-%m-%d"), From 468a01d08b50f3269a0547b4475e88fa7b625955 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Sat, 30 Apr 2022 17:08:25 +0200 Subject: [PATCH 02/14] restore and added stream_test in unit test --- .../source-tiktok-marketing/unit_tests/streams_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py index 5ecd2d9892b9..889117b5e319 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py @@ -26,6 +26,7 @@ "secret": "secret", "authenticator": None, "start_date": START_DATE, + "end_date": END_DATE, "app_id": 1234, "advertiser_id": 0, } @@ -34,6 +35,7 @@ "secret": "secret", "authenticator": None, "start_date": START_DATE, + "end_date": END_DATE, "app_id": 1234, "advertiser_id": 2000, } @@ -62,13 +64,13 @@ def advertiser_ids_fixture(): ], ) def test_get_time_interval(pendulum_now_mock, granularity, intervals_len): - intervals = BasicReports._get_time_interval(start_date="2020-01-01", granularity=granularity) + intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity) assert len(list(intervals)) == intervals_len @patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25")) def test_get_time_interval_past(pendulum_now_mock_past): - intervals = BasicReports._get_time_interval(start_date="2020-01-01", granularity=ReportGranularity.DAY) + intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY) assert len(list(intervals)) == 1 From 30fc336b7b06f6a7e91e0182a1da8299559a03ce Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Sun, 1 May 2022 02:24:13 +0200 Subject: [PATCH 03/14] updated spec.json --- .../source-tiktok-marketing/integration_tests/spec.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index da6cad26a536..ffe2de259820 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -113,6 +113,14 @@ "enum": ["LIFETIME", "DAY", "HOUR"], "order": 2, "type": "string" + }, + "end_date": { + "title": "End Date", + "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", + "default": "2022-05-01", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 3, + "type": "string" } } }, From a0d01a1d8ac23e166dd53fb7d4b9088413334594 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 9 May 2022 09:41:48 +0200 Subject: [PATCH 04/14] updated with master --- .../advertiser_ids.yml | 74 +++++++++---------- .../integration_tests/spec.json | 2 +- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml index 1dfdff184e50..064ef6a7eb95 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml @@ -29,9 +29,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Sat, 30 Apr 2022 13:41:16 GMT + - Wed, 04 May 2022 10:48:03 GMT Expires: - - Sat, 30 Apr 2022 13:41:16 GMT + - Wed, 04 May 2022 10:48:03 GMT Location: - https://ads.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 Pragma: @@ -39,24 +39,24 @@ interactions: Server: - nginx Server-Timing: - - cdn-cache; desc=MISS, edge; dur=84, origin; dur=10 - - inner; dur=2 + - cdn-cache; desc=MISS, edge; dur=80, origin; dur=2 + - inner; dur=1 X-Akamai-Request-ID: - - 6c168ba1.3225038f + - e1d0c7b.1b2e3154 X-Cache: - - TCP_MISS from a88-221-221-133.deploy.akamaitechnologies.com (AkamaiGHost/10.7.5-41022941) + - TCP_MISS from a2-16-100-37.deploy.akamaitechnologies.com (AkamaiGHost/10.8.0-41078462) (-) X-Cache-Remote: - - TCP_MISS from a172-232-7-15.deploy.akamaitechnologies.com (AkamaiGHost/10.7.5-41022941) + - TCP_MISS from a23-220-107-15.deploy.akamaitechnologies.com (AkamaiGHost/10.7.5-41022941) (-) X-Origin-Response-Time: - - 10,172.232.7.15 + - 2,23.220.107.15 X-Parent-Response-Time: - - 94,88.221.221.133 + - 82,2.16.100.37 x-tt-logid: - - 202204301341160101131350911272485E + - 202205041048030101130060690F278C0D x-tt-trace-host: - - 0187fa220474ed9f3ce919185a4a99d3a80f27baa2153625a326437072514ab1e1ea27c3af023794a41bdb590603bf0ecef0aeb1b499ac24edb430501660a9a5f4fe2361247cd1065faf1e0e95223f5845cf8c378f5f9bc6a480d2486fef9df47edc8a7040d2204554f97fec059f253a7f + - 0187fa220474ed9f3ce919185a4a99d3a849c53a0c205d2c6cd5bdc4e5216e291338f15036208f402362dbf7a3489e51e569aa82e8655ea0ccf6f908e42e4fb839d19496350d0c54a958590b538a6d9a4d9833f7f68aeb720d08d5e7381108398e422875ee383845a909ccb50f974fc61d x-tt-trace-tag: - id=16;cdn-cache=miss;type=dyn status: @@ -78,23 +78,23 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//rJZdb+I4FIb/isXFalaaVsff9txRoKVDaStot9LurFaGGIiApGMnsyNV - /e97EqYZLvD2ZiQIsZ34Oef49Wteessy871PBD6S3t7H6NZNq3c36WFH8F9rH6t/8qzpY8AYCA6U - C0oVUGBCMkHxBgacciOgeSdzlcOnX3q7PFZ489dLz2XffKjy6MNhJmUkcMlBaGq5NsB4Qz96qnD7 - Noqxi8+OnJFxf9pc3X5Rh7UPZP7swtbF6En/Cge0MsCt6b1+JKdgSkiKEAmGSobR89Owz265JU/l - bhW3eYGzXgRXZB38c38wIU93N5fzyfUt6dcxfKkBVstY1cWarA6NQIbB1RhX8z6iQGCdUmEZaZsv - ALVYT6vs6bAud25ZFnkT0GiG17f21X4xxqbglhnJk8lbbrQQCuutpTRMJ5Jv4s9UM/90OOuSnl4/ - PIxuhqPHh/lgjPTZ4+3w8vF2gkOoAC0ZZQmupfgRRkmluOJWygR3/rV2wZNRkX/vqMd9P7LUAIYl - WZgi6tLiL2ZKUYqnWdO724dRR/mzrCoyH5HfyKA8J5NGR/iykMImOYIZBtZqygCBMsUZliGvytiR - 7v1zzAclGfq6isvNrpFVm9eXupEjEZJrzY36Hyy3oI1SRivMTp/GzsrlNlYuvM89exeJUgGrqBKU - MW20kaeRLWHqvpNfwVQcrQW4BpSOACkSzNVB/6lZDMrBSsqU1VYbY+jpWQaX913MzX272SOZH/Zx - uxtW/6IKiw32+108JHGkFcok0zoVhRUCXU0BaEbRHlNR9BeLRoW40frXeP3RfKsXCIMGSZMMQxlK - 0TAULXDBEvW6Kdd55ZebhvI4wGvX8cZRmpqEf2jgxgImCtqCNdYw9Z5/DF0koxAr39W3PxuS+1Cu - g9vvszz4bZWXBZkeSl0sN61bYj0lJOqJUWC2SkqqAf2EoWEmdrjbVR47cJmKyhcVGdeLn2b2NjjO - 0eOz3BexIv78j3Mc48AU5ihTfCOZEkJzQD7gfkx4dfQLvMu6BZ1j+3njwt4d6aeVj8Q9hU6SBirQ - Slsm0UM1GmkCeDLhD1co5ur3X5K4pcyq5qRi6OToCQnv+bn8T0fHxxP+d8gORoD0WV1kq7rYkkm7 - 8mrXLjseX2juCcs98I3F00RrptGQ2Hv82cVFx++IFz7s8uKs3eS+WOCfCBzVzdwcj+e/X1//AwAA - //8DAAZBQqwHCQAA + H4sIAAAAAAAAAwAAAP//rJbbbts4EIZfhfDFogs0wfBM9s6nxKnjJLCTDbDbxYK2aEewLaWk1C0Q + 5N13JDeqL8zNTQGbEkmJ38xw5qdeeqsy871PBD6S3t7H6DZNr3c77eFA8F9rH6t/8qwZY8AYSBAU + hAEBFJiQVEpGAQbScMlF807mKodPv/R2eazw5q+Xnsu++VDl0YfDSspIwMdBaGq5NsB4Qz96qnD7 + 1oqJi8+OnJFJf9a0br+sw8YHsnh2Yeti9KR/iRNaGeDW9F4/klMwhWYiRIKhkgHl/DTss1ttyWO5 + W8dtXuCqg+CKrIN/7g+n5PH2+mIxvboh/TqGLzXAehWrutiQ9aETyCi4Gu1q3kcUxkpByiwjbfMH + oFZQapU9bdbFzq3KIm8MGs+xfetf7pcT7ApumZE86bzlRguhMN5aSsN0wvnG/kw1689G887p2dX9 + /fh6NH64XwwnSJ8/3IwuHm6mOEWp0rj1LMG1FH/CKKkUV9xKmeAuvtYueDIu8u8d9Xjsh5cawLAk + C10EjANe0VPKaYI1u725H3eUP8uqIosx+Y0My3MybfIIXxZS2CRHMMPAWk0ZIFCmOKMy5FUZO9Kd + f475sCQjX1dx9bRr0qr160vdpCMRkmvNjfofLLegjVJGK/ROn8bOy9U2Vi68zz17F4mpAlZRJShj + 2mgjTyNbwsx9J7+CqTjHiuEaMHUESJFgrg/5n1rFYDpYSZmy2mpjDD29yvDirrO5uW+LPZLFoY7b + alj/i1lYPOG438WDE0e5QplkWqessEKgqikAzSjKY8qK/nLZZCEWWv8K2x/dt3ihzKJA0iTDUIap + aBgmLXDBEvG6Ljd55VdPDeVhiG038MZRmpqEfmjgxgI6CtqCNdYw9Z5+jFwk4xAr38W3Px+Ru1Bu + gtvvszz4bZWXBZkdQl2snlq1xHhKSMQTrUBvlZRUA+oJQ8FMVLjbVR4HcJuKyhcVmdTLn2L2NjnJ + UeOz3BexIv78j3Oc48AU+ihTfCOZEkJzQD5gPSa0Ovol3mXdhi6w//zkwt4d5U+bPhJrCpUkDVSg + lbZMooZqFNIE8KTDHy4xmavff4njljKrmpOKoZKjJiS05+f2Px4dH4/47ZAdhADp87rI1nWxJdN2 + 59Wu3XY8vlDcE5J74BuLp4nWTKMgsff488Gg43fEgQ+7vDhri9wXS/yIwFndrM3xeP779fU/AAAA + //8DAIEkycYHCQAA headers: Connection: - keep-alive @@ -105,12 +105,12 @@ interactions: Content-Type: - application/json Date: - - Sat, 30 Apr 2022 13:41:16 GMT + - Wed, 04 May 2022 10:48:04 GMT Server: - nginx Server-Timing: - - cdn-cache; desc=MISS, edge; dur=173, origin; dur=204 - - inner; dur=202 + - cdn-cache; desc=MISS, edge; dur=154, origin; dur=60 + - inner; dur=53 Vary: - Accept-Encoding X-Domain-Change-Info: @@ -118,20 +118,20 @@ interactions: on 09/30/2021. Please update all of your requests with the new domain if you haven't done so already. For more information, please visit https://ads.tiktok.com/marketing_api/docs?rid=esb88xalm6m&id=1709207085043713 X-Origin-Response-Time: - - 204,23.44.4.15 + - 60,23.61.0.115 X-Parent-Response-Time: - - 372,88.221.221.133 + - 213,2.16.100.37 X-TT-LOGID: - - 202204301341160102452410100C313840 - - 202204301341160102452410100C313840 + - 202205041048040102451552100B583534 + - 202205041048040102451552100B583534 X_TT_LOGID: - - 202204301341160102452410100C313840 + - 202205041048040102451552100B583534 stablecode: - '0' upstream-caught: - None x-tt-trace-host: - - 0187fa220474ed9f3ce919185a4a99d3a80f27baa2153625a326437072514ab1e1843afc31012de609bffa64a8a8247d7427ab22037339612b2c7d5b2c73861bbb62cec17bafa555777465bd3758eea26daacab5ba904e00a94f47bddb6c63c2ef37b36f4b8bdb136e69e32956a8584f8b + - 0187fa220474ed9f3ce919185a4a99d3a849c53a0c205d2c6cd5bdc4e5216e2913c39bec8c59efd1818e8ac8f081098943e42630d94e3ac0dbf7e837dd0051c1abe888f65d1f22faba9858651ec4c879fed92abeb3f022990d571c1e3cd77449c5820d033f9f7397d9730edb4b97f09be6 x-tt-trace-tag: - id=16;cdn-cache=miss;type=dyn status: diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index ffe2de259820..cb9fe6e87856 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -117,7 +117,7 @@ "end_date": { "title": "End Date", "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "default": "2022-05-01", + "default": "2022-05-04", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "order": 3, "type": "string" From b90dfd48c2cc40493130cfe8c8f8efeb10833579 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 9 May 2022 14:44:59 +0200 Subject: [PATCH 05/14] spec.json updated --- .../advertiser_ids.yml | 77 +++++++++---------- .../integration_tests/spec.json | 2 +- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml index 064ef6a7eb95..07dfab2f6a04 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml @@ -29,9 +29,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Wed, 04 May 2022 10:48:03 GMT + - Mon, 09 May 2022 11:54:10 GMT Expires: - - Wed, 04 May 2022 10:48:03 GMT + - Mon, 09 May 2022 11:54:10 GMT Location: - https://ads.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 Pragma: @@ -39,24 +39,19 @@ interactions: Server: - nginx Server-Timing: - - cdn-cache; desc=MISS, edge; dur=80, origin; dur=2 - - inner; dur=1 + - inner; dur=3 + - cdn-cache; desc=MISS, edge; dur=15, origin; dur=105 X-Akamai-Request-ID: - - e1d0c7b.1b2e3154 + - 104915bf X-Cache: - - TCP_MISS from a2-16-100-37.deploy.akamaitechnologies.com (AkamaiGHost/10.8.0-41078462) - (-) - X-Cache-Remote: - - TCP_MISS from a23-220-107-15.deploy.akamaitechnologies.com (AkamaiGHost/10.7.5-41022941) + - TCP_MISS from a88-221-221-109.deploy.akamaitechnologies.com (AkamaiGHost/10.8.0-41078462) (-) X-Origin-Response-Time: - - 2,23.220.107.15 - X-Parent-Response-Time: - - 82,2.16.100.37 + - 105,88.221.221.109 x-tt-logid: - - 202205041048030101130060690F278C0D + - 20220509115410010113006104212BE2DC x-tt-trace-host: - - 0187fa220474ed9f3ce919185a4a99d3a849c53a0c205d2c6cd5bdc4e5216e291338f15036208f402362dbf7a3489e51e569aa82e8655ea0ccf6f908e42e4fb839d19496350d0c54a958590b538a6d9a4d9833f7f68aeb720d08d5e7381108398e422875ee383845a909ccb50f974fc61d + - 0187fa220474ed9f3ce919185a4a99d3a85c8911db5690a876694b55e309216ace0ddbd7d3aff37a297bb8be22108156f0700ecaef99e0095c7b6da7ef72dfc622147867f6017142897e3d28bf5d8698591244dfd24c531039fbd78054f403507d x-tt-trace-tag: - id=16;cdn-cache=miss;type=dyn status: @@ -78,39 +73,39 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//rJbbbts4EIZfhfDFogs0wfBM9s6nxKnjJLCTDbDbxYK2aEewLaWk1C0Q - 5N13JDeqL8zNTQGbEkmJ38xw5qdeeqsy871PBD6S3t7H6DZNr3c77eFA8F9rH6t/8qwZY8AYSBAU - hAEBFJiQVEpGAQbScMlF807mKodPv/R2eazw5q+Xnsu++VDl0YfDSspIwMdBaGq5NsB4Qz96qnD7 - 1oqJi8+OnJFJf9a0br+sw8YHsnh2Yeti9KR/iRNaGeDW9F4/klMwhWYiRIKhkgHl/DTss1ttyWO5 - W8dtXuCqg+CKrIN/7g+n5PH2+mIxvboh/TqGLzXAehWrutiQ9aETyCi4Gu1q3kcUxkpByiwjbfMH - oFZQapU9bdbFzq3KIm8MGs+xfetf7pcT7ApumZE86bzlRguhMN5aSsN0wvnG/kw1689G887p2dX9 - /fh6NH64XwwnSJ8/3IwuHm6mOEWp0rj1LMG1FH/CKKkUV9xKmeAuvtYueDIu8u8d9Xjsh5cawLAk - C10EjANe0VPKaYI1u725H3eUP8uqIosx+Y0My3MybfIIXxZS2CRHMMPAWk0ZIFCmOKMy5FUZO9Kd - f475sCQjX1dx9bRr0qr160vdpCMRkmvNjfofLLegjVJGK/ROn8bOy9U2Vi68zz17F4mpAlZRJShj - 2mgjTyNbwsx9J7+CqTjHiuEaMHUESJFgrg/5n1rFYDpYSZmy2mpjDD29yvDirrO5uW+LPZLFoY7b - alj/i1lYPOG438WDE0e5QplkWqessEKgqikAzSjKY8qK/nLZZCEWWv8K2x/dt3ihzKJA0iTDUIap - aBgmLXDBEvG6Ljd55VdPDeVhiG038MZRmpqEfmjgxgI6CtqCNdYw9Z5+jFwk4xAr38W3Px+Ru1Bu - gtvvszz4bZWXBZkdQl2snlq1xHhKSMQTrUBvlZRUA+oJQ8FMVLjbVR4HcJuKyhcVmdTLn2L2NjnJ - UeOz3BexIv78j3Oc48AU+ihTfCOZEkJzQD5gPSa0Ovol3mXdhi6w//zkwt4d5U+bPhJrCpUkDVSg - lbZMooZqFNIE8KTDHy4xmavff4njljKrmpOKoZKjJiS05+f2Px4dH4/47ZAdhADp87rI1nWxJdN2 - 59Wu3XY8vlDcE5J74BuLp4nWTKMgsff488Gg43fEgQ+7vDhri9wXS/yIwFndrM3xeP779fU/AAAA - //8DAIEkycYHCQAA + H4sIAAAAAAAAA6yW227bOBCGX4XwxaILNMHwTPbOxzh1nAR2sgF2u1jQFm0LtqWUlLoFgrz7juxG + 9YW5uSlgUyIp8ZsZzvzUS2dZZr7zicBH0tn7GN266XXuJh0cCP5r7WP1T541YwwYAwmWUikoAAUm + JBOCWguaqR4fmeadzFUOn37p7PJY4c1fLx2XffOhyqMPx5WUkcAlB6Gp5doA4w395KnC7Q9WjF18 + duSCjLvTpnX7RR3WPpD5swtbF6Mn3Suc0MoAt6bz+pGcgykhKUIkGCoZUM7Pwz675ZY8lbtV3OYF + rtoLrsha+Oduf0Ke7m5G88n1LenWMXypAVbLWNXFmqyOnUAGwdVoV/M+okBQBSmzjLTNHwNpBaVW + 2fNmjXZuWRZ5Y9Bwhu1b/2q/GGNXcMuM5EnnLTdaCIXx1lIaphPON/Znqll/Opi1Tk+vHx6GN4Ph + 48O8P0b67PF2MHq8neAUpUpLRlmCayn+hFFSKa64lTLBnX+tXfBkWOTfW+rp2A8vNYBhSRa6CBgH + vKKnlNMEa3p3+zBsKX+WVUXmQ/Ib6ZeXZNLkEb4spLBJjmCGgbWaMkCgTHEGZcirMrake/8c835J + Br6u4nKza9Lq4NeXuklHIiTXmhv1P1iOJWaUMlqhd/o8dlYut7Fy4X3uxbtITBWwiipBGdNGG3ke + eSBM3XfyK5iKc6wYrgFTR4AUCebqmP+pVQymg5WUKautNsbQ86v0R/etzc39odgjmR/r+FANq38x + C4sNjvtdPDpxkiuUSaZ1ygorBKqaAhRGilKZsqK7WDRZiIXWvcb2R/ctXiAMCiRNMgxlmIqGYdIC + FywRr5tynVd+uWkoj31s24E3jtLUJPRDAzcW0FHQFqyxhqn39GPgIhmGWPk2vt3ZgNyHch3cfp/l + wW+rvCzI9BjqYrk5qCXGU0IinmgFequkpBpQTxgKZqLC3a7yOIDbVFS+qMi4XvwUs7fJcY4an+W+ + iBXxl39c4hwHptBHmeIbyZQQmgPyAesxodXRL/Auazd0jv3njQt7d5I/h/SRWFOoJGmgAq20ZRI1 + VKOQJoBnHf5whclc/f5LHLeUWdWcVAyVHDUhoT0/t//p5Ph4wm+H7CgESJ/VRbaqiy2ZHHZe7Q7b + jscXintCco98Y/E00ZppFCT2Hn/W67X8ltjzYZcXF4ci98UCPyJwVjdrczye/359/Q8AAP//AwDh + C7zBBwkAAA== headers: Connection: - keep-alive Content-Encoding: - gzip Content-Length: - - '924' + - '919' Content-Type: - application/json Date: - - Wed, 04 May 2022 10:48:04 GMT + - Mon, 09 May 2022 11:54:10 GMT Server: - nginx Server-Timing: - - cdn-cache; desc=MISS, edge; dur=154, origin; dur=60 - - inner; dur=53 + - cdn-cache; desc=MISS, edge; dur=160, origin; dur=57 + - inner; dur=54 Vary: - Accept-Encoding X-Domain-Change-Info: @@ -118,20 +113,20 @@ interactions: on 09/30/2021. Please update all of your requests with the new domain if you haven't done so already. For more information, please visit https://ads.tiktok.com/marketing_api/docs?rid=esb88xalm6m&id=1709207085043713 X-Origin-Response-Time: - - 60,23.61.0.115 + - 57,23.44.4.84 X-Parent-Response-Time: - - 213,2.16.100.37 + - 217,88.221.221.109 X-TT-LOGID: - - 202205041048040102451552100B583534 - - 202205041048040102451552100B583534 + - 202205091154100102452441990726B3F8 + - 202205091154100102452441990726B3F8 X_TT_LOGID: - - 202205041048040102451552100B583534 + - 202205091154100102452441990726B3F8 stablecode: - '0' upstream-caught: - None x-tt-trace-host: - - 0187fa220474ed9f3ce919185a4a99d3a849c53a0c205d2c6cd5bdc4e5216e2913c39bec8c59efd1818e8ac8f081098943e42630d94e3ac0dbf7e837dd0051c1abe888f65d1f22faba9858651ec4c879fed92abeb3f022990d571c1e3cd77449c5820d033f9f7397d9730edb4b97f09be6 + - 0187fa220474ed9f3ce919185a4a99d3a8e996518fc1d65fa5113a443513eda0108f68b705ddc3b60486eb7bb41502c1097224ac8ffaadb632ec43bae7c134fbab127d4ccfacda115a0e88d3b3e43cc82c1b9b43a05cb48ed359981763a1b3ddc630046040694a10c62fd08fd35e09cc23 x-tt-trace-tag: - id=16;cdn-cache=miss;type=dyn status: diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index cb9fe6e87856..920ea1c32a93 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -117,7 +117,7 @@ "end_date": { "title": "End Date", "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "default": "2022-05-04", + "default": "2022-05-09", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "order": 3, "type": "string" From dfb29890bfd9a87ac3b382d4a99eb6f6c54a8049 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Tue, 10 May 2022 23:53:44 +0200 Subject: [PATCH 06/14] updated streams_test.py --- .../advertiser_ids.yml | 135 ------------------ .../unit_tests/streams_test.py | 12 +- 2 files changed, 6 insertions(+), 141 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml b/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml deleted file mode 100644 index 07dfab2f6a04..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/advertiser_ids.yml +++ /dev/null @@ -1,135 +0,0 @@ -interactions: -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - python-requests/2.27.1 - method: GET - uri: https://business-api.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 - response: - body: - string: 'Temporary - Redirect. - - - ' - headers: - Cache-Control: - - max-age=0, no-cache, no-store - Connection: - - keep-alive - Content-Length: - - '232' - Content-Type: - - text/html; charset=utf-8 - Date: - - Mon, 09 May 2022 11:54:10 GMT - Expires: - - Mon, 09 May 2022 11:54:10 GMT - Location: - - https://ads.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 - Pragma: - - no-cache - Server: - - nginx - Server-Timing: - - inner; dur=3 - - cdn-cache; desc=MISS, edge; dur=15, origin; dur=105 - X-Akamai-Request-ID: - - 104915bf - X-Cache: - - TCP_MISS from a88-221-221-109.deploy.akamaitechnologies.com (AkamaiGHost/10.8.0-41078462) - (-) - X-Origin-Response-Time: - - 105,88.221.221.109 - x-tt-logid: - - 20220509115410010113006104212BE2DC - x-tt-trace-host: - - 0187fa220474ed9f3ce919185a4a99d3a85c8911db5690a876694b55e309216ace0ddbd7d3aff37a297bb8be22108156f0700ecaef99e0095c7b6da7ef72dfc622147867f6017142897e3d28bf5d8698591244dfd24c531039fbd78054f403507d - x-tt-trace-tag: - - id=16;cdn-cache=miss;type=dyn - status: - code: 307 - message: Temporary Redirect -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - python-requests/2.27.1 - method: GET - uri: https://ads.tiktok.com/open_api/v1.2/oauth2/advertiser/get/?access_token=eb84ec191e411910dd399658868285769d63bb74&secret=27325d9aeba472ee3488ca19fba858d0b9256528&app_id=7059398826003529729 - response: - body: - string: !!binary | - H4sIAAAAAAAAA6yW227bOBCGX4XwxaILNMHwTPbOxzh1nAR2sgF2u1jQFm0LtqWUlLoFgrz7juxG - 9YW5uSlgUyIp8ZsZzvzUS2dZZr7zicBH0tn7GN266XXuJh0cCP5r7WP1T541YwwYAwmWUikoAAUm - JBOCWguaqR4fmeadzFUOn37p7PJY4c1fLx2XffOhyqMPx5WUkcAlB6Gp5doA4w395KnC7Q9WjF18 - duSCjLvTpnX7RR3WPpD5swtbF6Mn3Suc0MoAt6bz+pGcgykhKUIkGCoZUM7Pwz675ZY8lbtV3OYF - rtoLrsha+Oduf0Ke7m5G88n1LenWMXypAVbLWNXFmqyOnUAGwdVoV/M+okBQBSmzjLTNHwNpBaVW - 2fNmjXZuWRZ5Y9Bwhu1b/2q/GGNXcMuM5EnnLTdaCIXx1lIaphPON/Znqll/Opi1Tk+vHx6GN4Ph - 48O8P0b67PF2MHq8neAUpUpLRlmCayn+hFFSKa64lTLBnX+tXfBkWOTfW+rp2A8vNYBhSRa6CBgH - vKKnlNMEa3p3+zBsKX+WVUXmQ/Ib6ZeXZNLkEb4spLBJjmCGgbWaMkCgTHEGZcirMrake/8c835J - Br6u4nKza9Lq4NeXuklHIiTXmhv1P1iOJWaUMlqhd/o8dlYut7Fy4X3uxbtITBWwiipBGdNGG3ke - eSBM3XfyK5iKc6wYrgFTR4AUCebqmP+pVQymg5WUKautNsbQ86v0R/etzc39odgjmR/r+FANq38x - C4sNjvtdPDpxkiuUSaZ1ygorBKqaAhRGilKZsqK7WDRZiIXWvcb2R/ctXiAMCiRNMgxlmIqGYdIC - FywRr5tynVd+uWkoj31s24E3jtLUJPRDAzcW0FHQFqyxhqn39GPgIhmGWPk2vt3ZgNyHch3cfp/l - wW+rvCzI9BjqYrk5qCXGU0IinmgFequkpBpQTxgKZqLC3a7yOIDbVFS+qMi4XvwUs7fJcY4an+W+ - iBXxl39c4hwHptBHmeIbyZQQmgPyAesxodXRL/Auazd0jv3njQt7d5I/h/SRWFOoJGmgAq20ZRI1 - VKOQJoBnHf5whclc/f5LHLeUWdWcVAyVHDUhoT0/t//p5Ph4wm+H7CgESJ/VRbaqiy2ZHHZe7Q7b - jscXintCco98Y/E00ZppFCT2Hn/W67X8ltjzYZcXF4ci98UCPyJwVjdrczye/359/Q8AAP//AwDh - C7zBBwkAAA== - headers: - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Length: - - '919' - Content-Type: - - application/json - Date: - - Mon, 09 May 2022 11:54:10 GMT - Server: - - nginx - Server-Timing: - - cdn-cache; desc=MISS, edge; dur=160, origin; dur=57 - - inner; dur=54 - Vary: - - Accept-Encoding - X-Domain-Change-Info: - - We have moved to a new domain for API services after the Platform Upgrade - on 09/30/2021. Please update all of your requests with the new domain if you - haven't done so already. For more information, please visit https://ads.tiktok.com/marketing_api/docs?rid=esb88xalm6m&id=1709207085043713 - X-Origin-Response-Time: - - 57,23.44.4.84 - X-Parent-Response-Time: - - 217,88.221.221.109 - X-TT-LOGID: - - 202205091154100102452441990726B3F8 - - 202205091154100102452441990726B3F8 - X_TT_LOGID: - - 202205091154100102452441990726B3F8 - stablecode: - - '0' - upstream-caught: - - None - x-tt-trace-host: - - 0187fa220474ed9f3ce919185a4a99d3a8e996518fc1d65fa5113a443513eda0108f68b705ddc3b60486eb7bb41502c1097224ac8ffaadb632ec43bae7c134fbab127d4ccfacda115a0e88d3b3e43cc82c1b9b43a05cb48ed359981763a1b3ddc630046040694a10c62fd08fd35e09cc23 - x-tt-trace-tag: - - id=16;cdn-cache=miss;type=dyn - status: - code: 200 - message: OK -version: 1 diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py index 889117b5e319..5bb30c08e9a8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py @@ -22,21 +22,21 @@ START_DATE = "2020-01-01" END_DATE = "2020-03-01" CONFIG = { - "access_token": "access_token", - "secret": "secret", + "access_token": "00684a976b98efc4b4197e466c4ac71e986e8bab", + "secret": "27325d9aeba472ee3488ca19fba858d0b9256528", "authenticator": None, "start_date": START_DATE, "end_date": END_DATE, - "app_id": 1234, + "app_id": 7059398826003529729, "advertiser_id": 0, } CONFIG_SANDBOX = { - "access_token": "access_token", - "secret": "secret", + "access_token": "00684a976b98efc4b4197e466c4ac71e986e8bab", + "secret": "27325d9aeba472ee3488ca19fba858d0b9256528", "authenticator": None, "start_date": START_DATE, "end_date": END_DATE, - "app_id": 1234, + "app_id": 7059398826003529729, "advertiser_id": 2000, } ADV_IDS = [{"advertiser_id": 1}, {"advertiser_id": 2}] From a84093c0a1804d251b13ea5d9dc5c5042966a6d4 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Fri, 13 May 2022 13:01:52 +0200 Subject: [PATCH 07/14] updated spec.json with new end date --- .../source-tiktok-marketing/integration_tests/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index 920ea1c32a93..dd47a1c65351 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -117,7 +117,7 @@ "end_date": { "title": "End Date", "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "default": "2022-05-09", + "default": "2022-05-13", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "order": 3, "type": "string" From 84e43b3e73db6f4aee1ec139a54a7726f14ec1d9 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 16 May 2022 09:04:50 +0200 Subject: [PATCH 08/14] updated in unit test file --- .../connectors/source-tiktok-marketing/unit_tests/unit_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index 5860cd09c430..a2978b5ce23c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -112,6 +112,7 @@ def test_random_items(prepared_prod_args): ) if not max_updated_value or max_updated_value < ad_items[-1][stream.cursor_field]: max_updated_value = ad_items[-1][stream.cursor_field] + # mock for ads for page, page_response in generate_pages(items=ad_items, page_size=page_size, last_empty=True): uri = f"/open_api/v1.2/ad/get/?page_size={page_size}&advertiser_id={advertiser_id}" From 6ea02d019af537f9f41cd78badcc8dfc80edb4d7 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Tue, 17 May 2022 16:07:42 +0200 Subject: [PATCH 09/14] stream test updated --- .../unit_tests/streams_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py index 5bb30c08e9a8..889117b5e319 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py @@ -22,21 +22,21 @@ START_DATE = "2020-01-01" END_DATE = "2020-03-01" CONFIG = { - "access_token": "00684a976b98efc4b4197e466c4ac71e986e8bab", - "secret": "27325d9aeba472ee3488ca19fba858d0b9256528", + "access_token": "access_token", + "secret": "secret", "authenticator": None, "start_date": START_DATE, "end_date": END_DATE, - "app_id": 7059398826003529729, + "app_id": 1234, "advertiser_id": 0, } CONFIG_SANDBOX = { - "access_token": "00684a976b98efc4b4197e466c4ac71e986e8bab", - "secret": "27325d9aeba472ee3488ca19fba858d0b9256528", + "access_token": "access_token", + "secret": "secret", "authenticator": None, "start_date": START_DATE, "end_date": END_DATE, - "app_id": 7059398826003529729, + "app_id": 1234, "advertiser_id": 2000, } ADV_IDS = [{"advertiser_id": 1}, {"advertiser_id": 2}] From d66899a55a9ada811dadef16b9ed42046dc60b81 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Mon, 23 May 2022 16:39:01 +0530 Subject: [PATCH 10/14] fix: formatting and spec --- .../integration_tests/spec.json | 339 +++++++++--------- .../streams_audience_incremental.json | 3 +- .../source_tiktok_marketing/source.py | 2 +- .../source_tiktok_marketing/spec.py | 2 +- .../source_tiktok_marketing/streams.py | 6 +- 5 files changed, 177 insertions(+), 175 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index dd47a1c65351..54f82e8efbb6 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -1,181 +1,184 @@ { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "connectionSpecification": { - "title": "TikTok Marketing Source Spec", - "type": "object", - "properties": { - "credentials": { - "title": "Authentication *", - "default": {}, - "order": 0, - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" + "type": "SPEC", + "spec": { + "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", + "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", + "connectionSpecification": { + "title": "TikTok Marketing Source Spec", + "type": "object", + "properties": { + "credentials": { + "title": "Authentication *", + "default": {}, + "order": 0, + "type": "object", + "oneOf": [ + { + "title": "OAuth2.0", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "oauth2.0", + "order": 0, + "type": "string" + }, + "app_id": { + "title": "App ID", + "description": "The App ID applied by the developer.", + "airbyte_secret": true, + "type": "string" + }, + "secret": { + "title": "Secret", + "description": "The private key of the developer's application.", + "airbyte_secret": true, + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "Long-term Authorized Access Token.", + "airbyte_secret": true, + "type": "string" + } }, - "app_id": { - "title": "App ID", - "description": "The App ID applied by the developer.", - "airbyte_secret": true, - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The private key of the developer's application.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } + "required": ["app_id", "secret", "access_token"] }, - "required": ["app_id", "secret", "access_token"] - }, - { - "title": "Production Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "prod_access_token", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The App ID applied by the developer.", - "type": "string" + { + "title": "Production Access Token", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "prod_access_token", + "order": 0, + "type": "string" + }, + "app_id": { + "title": "App ID", + "description": "The App ID applied by the developer.", + "type": "string" + }, + "secret": { + "title": "Secret", + "description": "The private key of the developer application.", + "airbyte_secret": true, + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "The Long-term Authorized Access Token.", + "airbyte_secret": true, + "type": "string" + } }, - "secret": { - "title": "Secret", - "description": "The private key of the developer application.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } + "required": ["app_id", "secret", "access_token"] }, - "required": ["app_id", "secret", "access_token"] - }, - { - "title": "Sandbox Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "sandbox_access_token", - "order": 0, - "type": "string" + { + "title": "Sandbox Access Token", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "sandbox_access_token", + "order": 0, + "type": "string" + }, + "advertiser_id": { + "title": "Advertiser ID", + "description": "The Advertiser ID which generated for the developer's Sandbox application.", + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "The Long-term Authorized Access Token.", + "airbyte_secret": true, + "type": "string" + } }, - "advertiser_id": { - "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["advertiser_id", "access_token"] - } - ] - }, - "start_date": { - "title": "Start Date *", - "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", - "default": "2016-09-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1, - "type": "string" - }, - "report_granularity": { - "title": "Report Granularity *", - "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.", - "default": "DAY", - "enum": ["LIFETIME", "DAY", "HOUR"], - "order": 2, - "type": "string" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "default": "2022-05-13", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 3, - "type": "string" - } - } - }, - "supportsIncremental": true, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "title": "CompleteOauthOutputSpecification", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "path_in_connector_config": ["credentials", "access_token"], - "type": "string" - } + "required": ["advertiser_id", "access_token"] + } + ] + }, + "start_date": { + "title": "Start Date *", + "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", + "default": "2016-09-01", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 1, + "type": "string" }, - "required": ["access_token"] - }, - "complete_oauth_server_input_specification": { - "title": "CompleteOauthServerInputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "type": "string" + "report_granularity": { + "title": "Report Granularity *", + "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.", + "default": "DAY", + "enum": ["LIFETIME", "DAY", "HOUR"], + "order": 2, + "type": "string" + }, + "end_date": { + "title": "End Date", + "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", + "default": "2022-05-23", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 3, + "type": "string" + } + } + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "title": "CompleteOauthOutputSpecification", + "type": "object", + "properties": { + "access_token": { + "title": "Access Token", + "path_in_connector_config": ["credentials", "access_token"], + "type": "string" + } }, - "secret": { - "title": "Secret", - "type": "string" - } + "required": ["access_token"] }, - "required": ["app_id", "secret"] - }, - "complete_oauth_server_output_specification": { - "title": "CompleteOauthServerOutputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "path_in_connector_config": ["credentials", "app_id"], - "type": "string" + "complete_oauth_server_input_specification": { + "title": "CompleteOauthServerInputSpecification", + "type": "object", + "properties": { + "app_id": { + "title": "App Id", + "type": "string" + }, + "secret": { + "title": "Secret", + "type": "string" + } }, - "secret": { - "title": "Secret", - "path_in_connector_config": ["credentials", "secret"], - "type": "string" - } + "required": ["app_id", "secret"] }, - "required": ["app_id", "secret"] + "complete_oauth_server_output_specification": { + "title": "CompleteOauthServerOutputSpecification", + "type": "object", + "properties": { + "app_id": { + "title": "App Id", + "path_in_connector_config": ["credentials", "app_id"], + "type": "string" + }, + "secret": { + "title": "Secret", + "path_in_connector_config": ["credentials", "secret"], + "type": "string" + } + }, + "required": ["app_id", "secret"] + } } - } - }, - "additionalProperties": true + }, + "additionalProperties": true + } } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_audience_incremental.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_audience_incremental.json index 92d2b3d4514a..b2367b2a8841 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_audience_incremental.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_audience_incremental.json @@ -2,7 +2,6 @@ "streams": [ { "stream": { - "name": "ad_group_audience_reports", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], @@ -35,4 +34,4 @@ "destination_sync_mode": "append" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index d93425f65dc8..ba2d01ab6d5a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -18,8 +18,8 @@ SourceTiktokMarketingSpec, ) from .streams import ( - DEFAULT_START_DATE, DEFAULT_END_DATE, + DEFAULT_START_DATE, AdGroupAudienceReports, AdGroups, AdGroupsReports, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py index f89a8d976fb0..907b9762396b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py @@ -10,7 +10,7 @@ from jsonschema import RefResolver from pydantic import BaseModel, Field -from .streams import DEFAULT_START_DATE, DEFAULT_END_DATE, ReportGranularity +from .streams import DEFAULT_END_DATE, DEFAULT_START_DATE, ReportGranularity class OauthCredSpec(BaseModel): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index a2f8cae2930f..def8f5fb75b6 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -426,9 +426,9 @@ def cursor_field(self): return [] @staticmethod - def _get_time_interval(start_date: Union[datetime, str], - ending_date: Union[datetime, str], - granularity: ReportGranularity) -> Iterable[Tuple[datetime, datetime]]: + def _get_time_interval( + start_date: Union[datetime, str], ending_date: Union[datetime, str], granularity: ReportGranularity + ) -> Iterable[Tuple[datetime, datetime]]: """Due to time range restrictions based on the level of granularity of reports, we have to chunk API calls in order to get the desired time range. Docs: https://ads.tiktok.com/marketing_api/docs?id=1714590313280513 From 9c9f3eef0859da7b9242730dcc87f0d622b7b725 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Mon, 23 May 2022 16:41:25 +0530 Subject: [PATCH 11/14] chore: updated the source-pipedrive version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-tiktok-marketing/Dockerfile | 2 +- docs/integrations/sources/tiktok-marketing.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2da12aedfb70..bc3a7da695f9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -799,7 +799,7 @@ - name: TikTok Marketing sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 dockerRepository: airbyte/source-tiktok-marketing - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing icon: tiktok.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 5524acd9717b..9c013f23f79d 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 9ae8455bfcad..ab3d4c3b4cf2 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -521,6 +521,7 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| 0.1.11 | 2022-04-27 | [12838](https://github.com/airbytehq/airbyte/pull/12838) | Added end date configuration for tiktok | | 0.1.7 | 2022-04-27 | [12380](https://github.com/airbytehq/airbyte/pull/12380) | fixed spec descriptions and documentation | | 0.1.6 | 2022-04-19 | [11378](https://github.com/airbytehq/airbyte/pull/11378) | updated logic for stream initializations, fixed errors in schemas, updated SAT and unit tests | | 0.1.5 | 2022-02-17 | [10398](https://github.com/airbytehq/airbyte/pull/10398) | Add Audience reports | From 49ab5cf3bbee2c80e8cd04a8c5041a2b9f2c291b Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Mon, 23 May 2022 17:25:10 +0530 Subject: [PATCH 12/14] fix: formatting and spec --- .../source-tiktok-marketing/integration_tests/spec.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index 54f82e8efbb6..e20f08ad51d1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -1,6 +1,4 @@ -{ - "type": "SPEC", - "spec": { + "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", "connectionSpecification": { @@ -181,4 +179,4 @@ }, "additionalProperties": true } -} + \ No newline at end of file From e3b458b9f54060afae247487f5b8285b3ea62f7b Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Mon, 23 May 2022 17:52:26 +0530 Subject: [PATCH 13/14] fix: formatting and spec --- .../integration_tests/spec.json | 341 +++++++++--------- 1 file changed, 170 insertions(+), 171 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index e20f08ad51d1..1800868a4102 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -1,182 +1,181 @@ - - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "connectionSpecification": { - "title": "TikTok Marketing Source Spec", - "type": "object", - "properties": { - "credentials": { - "title": "Authentication *", - "default": {}, - "order": 0, - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The App ID applied by the developer.", - "airbyte_secret": true, - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The private key of the developer's application.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", + "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", + "connectionSpecification": { + "title": "TikTok Marketing Source Spec", + "type": "object", + "properties": { + "credentials": { + "title": "Authentication *", + "default": {}, + "order": 0, + "type": "object", + "oneOf": [ + { + "title": "OAuth2.0", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "oauth2.0", + "order": 0, + "type": "string" }, - "required": ["app_id", "secret", "access_token"] + "app_id": { + "title": "App ID", + "description": "The App ID applied by the developer.", + "airbyte_secret": true, + "type": "string" + }, + "secret": { + "title": "Secret", + "description": "The private key of the developer's application.", + "airbyte_secret": true, + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "Long-term Authorized Access Token.", + "airbyte_secret": true, + "type": "string" + } }, - { - "title": "Production Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "prod_access_token", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The App ID applied by the developer.", - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The private key of the developer application.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } + "required": ["app_id", "secret", "access_token"] + }, + { + "title": "Production Access Token", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "prod_access_token", + "order": 0, + "type": "string" + }, + "app_id": { + "title": "App ID", + "description": "The App ID applied by the developer.", + "type": "string" }, - "required": ["app_id", "secret", "access_token"] + "secret": { + "title": "Secret", + "description": "The private key of the developer application.", + "airbyte_secret": true, + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "The Long-term Authorized Access Token.", + "airbyte_secret": true, + "type": "string" + } }, - { - "title": "Sandbox Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "sandbox_access_token", - "order": 0, - "type": "string" - }, - "advertiser_id": { - "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } + "required": ["app_id", "secret", "access_token"] + }, + { + "title": "Sandbox Access Token", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "sandbox_access_token", + "order": 0, + "type": "string" }, - "required": ["advertiser_id", "access_token"] - } - ] - }, - "start_date": { - "title": "Start Date *", - "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", - "default": "2016-09-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1, - "type": "string" - }, - "report_granularity": { - "title": "Report Granularity *", - "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.", - "default": "DAY", - "enum": ["LIFETIME", "DAY", "HOUR"], - "order": 2, - "type": "string" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "default": "2022-05-23", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 3, - "type": "string" - } + "advertiser_id": { + "title": "Advertiser ID", + "description": "The Advertiser ID which generated for the developer's Sandbox application.", + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "The Long-term Authorized Access Token.", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["advertiser_id", "access_token"] + } + ] + }, + "start_date": { + "title": "Start Date *", + "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", + "default": "2016-09-01", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 1, + "type": "string" + }, + "report_granularity": { + "title": "Report Granularity *", + "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.", + "default": "DAY", + "enum": ["LIFETIME", "DAY", "HOUR"], + "order": 2, + "type": "string" + }, + "end_date": { + "title": "End Date", + "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", + "default": "2022-05-23", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 3, + "type": "string" } - }, - "supportsIncremental": true, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "title": "CompleteOauthOutputSpecification", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "path_in_connector_config": ["credentials", "access_token"], - "type": "string" - } - }, - "required": ["access_token"] + } + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "title": "CompleteOauthOutputSpecification", + "type": "object", + "properties": { + "access_token": { + "title": "Access Token", + "path_in_connector_config": ["credentials", "access_token"], + "type": "string" + } }, - "complete_oauth_server_input_specification": { - "title": "CompleteOauthServerInputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "type": "string" - }, - "secret": { - "title": "Secret", - "type": "string" - } + "required": ["access_token"] + }, + "complete_oauth_server_input_specification": { + "title": "CompleteOauthServerInputSpecification", + "type": "object", + "properties": { + "app_id": { + "title": "App Id", + "type": "string" }, - "required": ["app_id", "secret"] + "secret": { + "title": "Secret", + "type": "string" + } }, - "complete_oauth_server_output_specification": { - "title": "CompleteOauthServerOutputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "path_in_connector_config": ["credentials", "app_id"], - "type": "string" - }, - "secret": { - "title": "Secret", - "path_in_connector_config": ["credentials", "secret"], - "type": "string" - } + "required": ["app_id", "secret"] + }, + "complete_oauth_server_output_specification": { + "title": "CompleteOauthServerOutputSpecification", + "type": "object", + "properties": { + "app_id": { + "title": "App Id", + "path_in_connector_config": ["credentials", "app_id"], + "type": "string" }, - "required": ["app_id", "secret"] - } + "secret": { + "title": "Secret", + "path_in_connector_config": ["credentials", "secret"], + "type": "string" + } + }, + "required": ["app_id", "secret"] } - }, - "additionalProperties": true - } - \ No newline at end of file + } + }, + "additionalProperties": true +} From 66e8cf08eef20bb8cc334270bc15726c3120a2b0 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Mon, 23 May 2022 19:26:40 +0530 Subject: [PATCH 14/14] fix: formatting and spec --- .../integration_tests/spec.json | 61 +++++-------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json index 1800868a4102..796cd56be510 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json @@ -6,7 +6,8 @@ "type": "object", "properties": { "credentials": { - "title": "Authentication *", + "title": "Authentication Method", + "description": "Authentication method", "default": {}, "order": 0, "type": "object", @@ -23,13 +24,13 @@ }, "app_id": { "title": "App ID", - "description": "The App ID applied by the developer.", + "description": "The Developer Application App ID.", "airbyte_secret": true, "type": "string" }, "secret": { "title": "Secret", - "description": "The private key of the developer's application.", + "description": "The Developer Application Secret.", "airbyte_secret": true, "type": "string" }, @@ -42,36 +43,6 @@ }, "required": ["app_id", "secret", "access_token"] }, - { - "title": "Production Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "prod_access_token", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The App ID applied by the developer.", - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The private key of the developer application.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["app_id", "secret", "access_token"] - }, { "title": "Sandbox Access Token", "type": "object", @@ -84,12 +55,12 @@ }, "advertiser_id": { "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", + "description": "The Advertiser ID which generated for the developer's Sandbox application.", "type": "string" }, "access_token": { "title": "Access Token", - "description": "The Long-term Authorized Access Token.", + "description": "The long-term authorized access token.", "airbyte_secret": true, "type": "string" } @@ -99,7 +70,7 @@ ] }, "start_date": { - "title": "Start Date *", + "title": "Replication Start Date *", "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", "default": "2016-09-01", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", @@ -107,8 +78,8 @@ "type": "string" }, "report_granularity": { - "title": "Report Granularity *", - "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.", + "title": "Report Aggregation Granularity *", + "description": "The granularity used for aggregating performance data in reports. See the docs.", "default": "DAY", "enum": ["LIFETIME", "DAY", "HOUR"], "order": 2, @@ -125,12 +96,16 @@ } }, "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "authSpecification": null, "advanced_auth": { "auth_flow_type": "oauth2.0", "predicate_key": ["credentials", "auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": null, "complete_oauth_output_specification": { "title": "CompleteOauthOutputSpecification", "type": "object", @@ -147,14 +122,8 @@ "title": "CompleteOauthServerInputSpecification", "type": "object", "properties": { - "app_id": { - "title": "App Id", - "type": "string" - }, - "secret": { - "title": "Secret", - "type": "string" - } + "app_id": { "title": "App Id", "type": "string" }, + "secret": { "title": "Secret", "type": "string" } }, "required": ["app_id", "secret"] },