🎉 Source Tiktok marketing - remove granularity config option (#13890)

* Removed granularity config option from spec, added corresponsing streams for each support granularity (hourly daily, lifetime), updated unittests, SAT * auto-formating * auto-formating * removed AdvertisersIds stream from list of exposed streams, updated docs * expose new style streams since 0.1.13, expose old streams for config for older version * update spec * fixed path to catalog * increased timeout
airbytehq · Jun 29, 2022 · 2904fde · 2904fde
1 parent 4fd3cb1
commit 2904fde
Show file tree

Hide file tree

Showing 21 changed files with 733 additions and 209 deletions.
diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml
@@ -9,14 +9,17 @@ tests:
   connection:
     - config_path: "secrets/prod_config.json"
       status: "succeed"
-    - config_path: "secrets/prod_config_day.json"
+    - config_path: "secrets/prod_config_with_day_granularity.json"
       status: "succeed"
-    - config_path: "secrets/config.json"
+    - config_path: "secrets/prod_config_with_lifetime_granularity.json"
       status: "succeed"
+# Sandbox creds are not working temporary because of issues from APi side
+#    - config_path: "secrets/config.json"
+#      status: "succeed"
+#    - config_path: "secrets/new_config_sandbox.json"
+#      status: "succeed"
     - config_path: "secrets/new_config_prod.json"
       status: "succeed"
-    - config_path: "secrets/new_config_sandbox.json"
-      status: "succeed"
     - config_path: "secrets/config_oauth.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
@@ -27,58 +30,54 @@ tests:
       status: "failed"
 
   discovery:
-    - config_path: "secrets/prod_config_day.json"
     - config_path: "secrets/prod_config.json"
+    - config_path: "secrets/prod_config_with_day_granularity.json"
+    - config_path: "secrets/prod_config_with_lifetime_granularity.json"
     - config_path: "secrets/config.json"
     - config_path: "secrets/new_config_prod.json"
     - config_path: "secrets/config_oauth.json"
 
   basic_read:
-    # DAY granularity:
-    - config_path: "secrets/prod_config_day.json"
-      configured_catalog_path: "integration_tests/streams_all.json"
+    # New style streams (for >= 0.1.13):
+    - config_path: "secrets/prod_config.json"
+      configured_catalog_path: "integration_tests/streams_basic.json"
+      timeout_seconds: 1200
+    - config_path: "secrets/prod_config.json"
+      configured_catalog_path: "integration_tests/streams_reports_daily.json"
       timeout_seconds: 1200
-    # LIFETIME granularity:
     - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_lifetime.json"
+      configured_catalog_path: "integration_tests/streams_reports_lifetime.json"
+      timeout_seconds: 1200
+    # Old style streams with granularity config (for < 0.1.13)
+    # Note: not needed to be tested separately in full and incremental tests, because code of
+    # these streams is called directly in new style streams
+    - config_path: "secrets/prod_config_with_day_granularity.json"
+      configured_catalog_path: "integration_tests/streams_with_day_granularity.json"
+      timeout_seconds: 1200
+    - config_path: "secrets/prod_config_with_lifetime_granularity.json"
+      configured_catalog_path: "integration_tests/streams_with_lifetime_granularity.json"
       timeout_seconds: 1200
 
   incremental:
-    # LIFETIME granularity: does not support incremental sync
-    # DAY granularity:
-    - config_path: "secrets/prod_config_day.json"
-      configured_catalog_path: "integration_tests/streams_all_incremental.json"
+    - config_path: "secrets/prod_config.json"
+      configured_catalog_path: "integration_tests/streams_basic.json"
+      timeout_seconds: 2400
       future_state_path: "integration_tests/abnormal_state.json"
-      timeout_seconds: 600
-      cursor_paths:
-        ads_reports: [ "dimensions", "stat_time_day" ]
-        advertisers_reports: [ "dimensions", "stat_time_day" ]
-        campaigns_reports: [ "dimensions", "stat_time_day" ]
-        ad_groups_reports: [ "dimensions", "stat_time_day" ]
-
-    # audience report streams have unexpected and inexplicable test failure caused by problem with schema (KeyError: 'properties'),
-    # It happens ONLY when incremental tests are run together with discovery tests.
-    # Error can't be reproduced by running test manually or ONLY incremental tests
-    - config_path: "secrets/prod_config_day.json"
-      configured_catalog_path: "integration_tests/streams_audience_incremental.json"
+    - config_path: "secrets/prod_config.json"
+      configured_catalog_path: "integration_tests/streams_reports_daily.json"
+      timeout_seconds: 2400
       future_state_path: "integration_tests/abnormal_state.json"
-      timeout_seconds: 600
-      cursor_paths:
-        ad_group_audience_reports: [ "dimensions", "stat_time_day" ]
-        ads_audience_reports: [ "dimensions", "stat_time_day" ]
-        advertisers_audience_reports: [ "dimensions", "stat_time_day" ]
-        campaigns_audience_reports_by_country: [ "dimensions", "stat_time_day" ]
+    # LIFETIME granularity: does not support incremental sync
 
   full_refresh:
-    # DAY granularity:
-    - config_path: "secrets/prod_config_day.json"
-      configured_catalog_path: "integration_tests/streams_all.json"
-      timeout_seconds: 6000
-      ignored_fields:  # Important: sometimes some streams does not return the same records in subsequent syncs
-        "ad_groups": ["dayparting", "enable_search_result", "display_mode", "schedule_infos", "feed_type", "status" ]
-    # LIFETIME granularity:
     - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_lifetime.json"
-      timeout_seconds: 600
+      configured_catalog_path: "integration_tests/streams_basic.json"
+      timeout_seconds: 2400
       ignored_fields:  # Important: sometimes some streams does not return the same records in subsequent syncs
         "ad_groups": ["dayparting", "enable_search_result", "display_mode", "schedule_infos", "feed_type", "status" ]
+    - config_path: "secrets/prod_config.json"
+      configured_catalog_path: "integration_tests/streams_reports_daily.json"
+      timeout_seconds: 2400
+    - config_path: "secrets/prod_config.json"
+      configured_catalog_path: "integration_tests/streams_reports_lifetime.json"
+      timeout_seconds: 2400
diff --git a/...yte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/...yte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json
@@ -9,43 +9,43 @@
     "modify_time": "2030-08-30 09:16:10"
   },
 
-  "ads_reports": {
+  "ads_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
-  "advertisers_reports": {
+  "advertisers_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
-  "ad_groups_reports": {
+  "ad_groups_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
-  "campaigns_reports": {
+  "campaigns_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
 
-  "ads_audience_reports": {
+  "ads_audience_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
-  "advertisers_audience_reports": {
+  "advertisers_audience_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
-  "ad_group_audience_reports": {
+  "ad_group_audience_reports_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }
   },
-  "campaigns_audience_reports_by_country": {
+  "campaigns_audience_reports_by_country_daily": {
     "dimensions": {
       "stat_time_day": "2030-01-01"
     }

diff --git a/...onnectors/source-tiktok-marketing/integration_tests/abnormal_state_with_granullarity.json b/...onnectors/source-tiktok-marketing/integration_tests/abnormal_state_with_granullarity.json
@@ -0,0 +1,53 @@
+{
+  "ads": {
+    "modify_time": "2030-08-30 09:16:10"
+  },
+  "ad_groups": {
+    "modify_time": "2030-08-30 09:16:10"
+  },
+  "campaigns": {
+    "modify_time": "2030-08-30 09:16:10"
+  },
+
+  "ads_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+  "advertisers_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+  "ad_groups_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+  "campaigns_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+
+  "ads_audience_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+  "advertisers_audience_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+  "ad_group_audience_reports": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  },
+  "campaigns_audience_reports_by_country": {
+    "dimensions": {
+      "stat_time_day": "2030-01-01"
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json
@@ -11,8 +11,7 @@
         "default": {},
         "order": 0,
         "type": "object",
-        "oneOf": [
-          {
+        "oneOf": [{
             "title": "OAuth2.0",
             "type": "object",
             "properties": {
@@ -42,8 +41,7 @@
               }
             },
             "required": ["app_id", "secret", "access_token"]
-          },
-          {
+          }, {
             "title": "Sandbox Access Token",
             "type": "object",
             "properties": {
@@ -77,34 +75,30 @@
         "order": 1,
         "type": "string"
       },
-      "report_granularity": {
-        "title": "Report Aggregation Granularity *",
-        "description": "The granularity used for aggregating performance data in reports. See <a href=\"https://docs.airbyte.com/integrations/sources/tiktok-marketing/#report-aggregation\">the docs</a>.",
-        "default": "DAY",
-        "enum": ["LIFETIME", "DAY", "HOUR"],
-        "order": 2,
-        "type": "string"
-      },
       "end_date": {
         "title": "End Date",
         "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.",
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        "order": 2,
+        "type": "string"
+      },
+      "report_granularity": {
+        "title": "Report Aggregation Granularity",
+        "description": "The granularity used for aggregating performance data in reports. See <a href=\"https://docs.airbyte.com/integrations/sources/tiktok-marketing/#report-aggregation\">the docs</a>.",
+        "enum": ["LIFETIME", "DAY", "HOUR"],
         "order": 3,
+        "airbyte_hidden": true,
         "type": "string"
       }
     }
   },
   "supportsIncremental": true,
-  "supportsNormalization": false,
-  "supportsDBT": false,
   "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"],
-  "authSpecification": null,
   "advanced_auth": {
     "auth_flow_type": "oauth2.0",
     "predicate_key": ["credentials", "auth_type"],
     "predicate_value": "oauth2.0",
     "oauth_config_specification": {
-      "oauth_user_input_from_connector_config_specification": null,
       "complete_oauth_output_specification": {
         "title": "CompleteOauthOutputSpecification",
         "type": "object",