Introduce new stream - campaign_insights_by_province

README.md

@@ -7,7 +7,7 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md).
 This tap: 
 
 - Pulls raw data from the [TikTok Marketing API](https://ads.tiktok.com/marketing_api/docs). 
-- Extracts the following resources from TikTok Marketing API for a single repository: 
+- Extracts the following resources from TikTok Marketing API for a single repository:
 	- [Accounts](https://ads.tiktok.com/marketing_api/docs?id=1739593083610113) - Endpoint: https://business-api.tiktok.com/open_api/v1.3/advertiser/info/ 
 	- [Campaigns](https://ads.tiktok.com/marketing_api/docs?id=1739315828649986) - Endpoint: https://ads.tiktok.com/open_api/v1.3/campaign/get/ 
 	- [Adgroups](https://ads.tiktok.com/marketing_api/docs?id=1739314558673922) - Endpoint: https://ads.tiktok.com/open_api/v1.3/adgroup/get/ 
@@ -17,6 +17,7 @@ This tap:
 		- [Ad Insights by Age and Gender](https://ads.tiktok.com/marketing_api/docs?id=1738864928947201)
 		- [Ad Insights by Country](https://ads.tiktok.com/marketing_api/docs?id=1738864928947201)
 		- [Ad Insights by Platform](https://ads.tiktok.com/marketing_api/docs?id=1738864928947201)
+        - [Campaign Insights by Province](https://ads.tiktok.com/marketing_api/docs?id=1738864928947201)
 - Outputs the schema for each resource
 - Incrementally pulls data based on the input state
 

tap_tiktok_ads/schemas/campaign_insights_by_province.json

@@ -0,0 +1,196 @@
+{
+  "type": [
+    "null",
+    "object"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "advertiser_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "stat_time_day": {
+      "type": [
+        "null",
+        "string"
+      ],
+      "format": "date-time"
+    },
+    "campaign_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "campaign_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "province_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "spend": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "cpc": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "cpm": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "impressions": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
+    "clicks": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
+    "ctr": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "conversion": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
+    "cost_per_conversion": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "conversion_rate": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "real_time_conversion": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
+    "real_time_cost_per_conversion": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "real_time_conversion_rate": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "result": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
+    "cost_per_result": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "result_rate": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "real_time_result": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
+    "real_time_cost_per_result": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "real_time_result_rate": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "gross_impressions": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "conversion_rate_v2": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "real_time_conversion_rate_v2": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "campaign_budget": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "campaign_dedicate_type": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "objective_type": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "split_test": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "rf_campaign_type": {
+      "type": [
+        "null",
+        "string"
+      ]
+    }
+  }
+}

tap_tiktok_ads/streams.py

@@ -9,7 +9,7 @@
 
 LOGGER = singer.get_logger()
 
-AUCTION_FIELDS = """[
+AUCTION_FIELDS = [
     "ad_name",
     "ad_text",
     "adgroup_id",
@@ -78,13 +78,8 @@
     "call_to_action",
     "image_mode",
     "billing_event"
-]"""
-AUDIENCE_FIELDS = """[
-    "ad_name",
-    "ad_text",
-    "adgroup_id",
-    "adgroup_name",
-    "campaign_id",
+]
+AUDIENCE_FIELDS = [
     "campaign_name",
     "spend",
     "cpc",
@@ -104,28 +99,36 @@
     "real_time_result",
     "real_time_cost_per_result",
     "real_time_result_rate",
-    "tt_app_id",
-    "tt_app_name",
-    "mobile_app_id",
-    "promotion_type",
-    "dpa_target_audience_type",
     "gross_impressions",
-    "is_smart_creative",
     "conversion_rate_v2",
     "real_time_conversion_rate_v2",
     "rf_campaign_type",
     "objective_type",
     "split_test",
     "campaign_budget",
-    "campaign_dedicate_type",
+    "campaign_dedicate_type"
+]
+AD_AUDIENCE_FIELDS = [
+    "ad_name",
+    "ad_text",
+    "adgroup_id",
+    "adgroup_name",
+    "campaign_id",
+    "billing_event",
+    "dpa_target_audience_type",
+    "is_smart_creative",
+    "mobile_app_id",
+    "promotion_type",
+    "tt_app_id",
+    "tt_app_name",
     "opt_status",
     "budget",
     "smart_target",
     "bid_strategy",
     "bid",
     "call_to_action",
-    "billing_event"
-]"""
+    "promotion_type",
+]
 ENDPOINT_ADVERTISERS = [
     'advertisers'
 ]
@@ -138,7 +141,8 @@
     'ad_insights',
     'ad_insights_by_age_and_gender',
     'ad_insights_by_country',
-    'ad_insights_by_platform'
+    'ad_insights_by_platform',
+    'campaign_insights_by_province'
 ]
 
 def get_date_batches(start_date, end_date):
@@ -427,7 +431,7 @@ class AdInsights(Insights):
             "ad_id",
             "stat_time_day"
         ]""",
-        "metrics": AUCTION_FIELDS,
+        "metrics": json.dumps(AUCTION_FIELDS),
         "query_lifetime": "false"
     }
 
@@ -446,7 +450,7 @@ class AdInsightsByAgeAndGender(Insights):
             "gender",
             "stat_time_day"
         ]""",
-        "metrics": AUDIENCE_FIELDS,
+        "metrics": json.dumps(AUDIENCE_FIELDS + AD_AUDIENCE_FIELDS),
         "query_lifetime": "false"
     }
 
@@ -464,7 +468,7 @@ class AdInsightsByCountry(Insights):
             "country_code",
             "stat_time_day"
         ]""",
-        "metrics": AUDIENCE_FIELDS,
+        "metrics": json.dumps(AUDIENCE_FIELDS + AD_AUDIENCE_FIELDS),
         "query_lifetime": "false"
     }
 
@@ -482,10 +486,28 @@ class AdInsightsByPlatform(Insights):
             "platform",
             "stat_time_day"
         ]""",
-        "metrics": AUDIENCE_FIELDS,
+        "metrics": json.dumps(AUDIENCE_FIELDS + AD_AUDIENCE_FIELDS),
         "query_lifetime": "false"
     }
 
+class CampaignInsightsByProvince(Insights):
+    tap_stream_id = "campaign_insights_by_province"
+    key_properties = ['advertiser_id', 'campaign_id', 'stat_time_day', 'province_id']
+    replication_keys  = ['stat_time_day']
+    path = "report/integrated/get/"
+    params = {
+        "service_type": "AUCTION",
+        "report_type": "AUDIENCE",
+        "data_level": "AUCTION_CAMPAIGN",
+        "dimensions": """[
+            "campaign_id",
+            "province_id",
+            "stat_time_day"
+        ]""",
+        "metrics": json.dumps(AUDIENCE_FIELDS),
+        "lifetime": "false"
+    }
+
 STREAMS = {
     'advertisers': Advertisers,
     'campaigns': Campaigns,
@@ -494,5 +516,6 @@ class AdInsightsByPlatform(Insights):
     'ad_insights': AdInsights,
     'ad_insights_by_age_and_gender': AdInsightsByAgeAndGender,
     'ad_insights_by_country': AdInsightsByCountry,
-    'ad_insights_by_platform': AdInsightsByPlatform
+    'ad_insights_by_platform': AdInsightsByPlatform,
+    'campaign_insights_by_province': CampaignInsightsByProvince
 }

tests/base.py

@@ -113,6 +113,12 @@ def expected_metadata(self):
                 self.REPLICATION_METHOD: self.INCREMENTAL,
                 self.REPLICATION_KEYS: {"stat_time_day"},
                 self.OBEYS_START_DATE: True
+            },
+            "campaign_insights_by_province": {
+                self.PRIMARY_KEYS: {"advertiser_id", "campaign_id", "stat_time_day", "province_id"},
+                self.REPLICATION_METHOD: self.INCREMENTAL,
+                self.REPLICATION_KEYS: {"stat_time_day"},
+                self.OBEYS_START_DATE: True
             }
         }
 

tests/test_pagination.py

@@ -20,7 +20,8 @@ def test_run(self):
         """
 
         # 'advertisers' records depends on the number of accounts passed for syncing
-        expected_streams = self.expected_streams() - self.unsupported_streams
+        # 'campaign_insights_by_province' has some API side issue. Returns 0 records for page size - 25
+        expected_streams = self.expected_streams() - self.unsupported_streams - {"campaign_insights_by_province"}
         conn_id = connections.ensure_connection(self)
 
         # Select all streams and all fields within streams

tests/test_start_date.py

@@ -34,7 +34,7 @@ def test_run(self):
         ### First Sync
         ##########################################################################
 
-        expected_streams = self.expected_streams() - { "advertisers", "ad_insights", "ad_insights_by_age_and_gender", "ad_insights_by_country", "ad_insights_by_platform"}
+        expected_streams = self.expected_streams() - self.unsupported_streams - {"campaign_insights_by_province"}
 
         conn_id_1 = connections.ensure_connection(self, original_properties=False)
         runner.run_check_mode(self, conn_id_1)