From 58044f9003c69cc3dd8fc164a2d49edb79e55800 Mon Sep 17 00:00:00 2001 From: Dan Mosora Date: Wed, 9 May 2018 10:27:10 -0400 Subject: [PATCH 1/3] First pass at data diff --- tap_zendesk/schemas/group-memberships.json | 6 +++-- tap_zendesk/schemas/groups.json | 6 +++-- tap_zendesk/schemas/macros.json | 6 +++-- tap_zendesk/schemas/organizations.json | 6 +++-- tap_zendesk/schemas/ticket-audits.json | 11 +++++---- tap_zendesk/schemas/ticket-fields.json | 6 +++-- tap_zendesk/schemas/ticket-metrics.json | 27 ++++++++++++++-------- tap_zendesk/schemas/tickets.json | 24 ++++++------------- tap_zendesk/schemas/users.json | 9 +++++--- tap_zendesk/streams.py | 6 +++-- 10 files changed, 61 insertions(+), 46 deletions(-) diff --git a/tap_zendesk/schemas/group-memberships.json b/tap_zendesk/schemas/group-memberships.json index 1b19cba..6d9660c 100644 --- a/tap_zendesk/schemas/group-memberships.json +++ b/tap_zendesk/schemas/group-memberships.json @@ -22,7 +22,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "group_id": { "type": [ @@ -34,7 +35,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "id": { "type": [ diff --git a/tap_zendesk/schemas/groups.json b/tap_zendesk/schemas/groups.json index b7c6ee5..795fab0 100644 --- a/tap_zendesk/schemas/groups.json +++ b/tap_zendesk/schemas/groups.json @@ -14,7 +14,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "url": { "type": [ @@ -26,7 +27,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "deleted": { "type": [ diff --git a/tap_zendesk/schemas/macros.json b/tap_zendesk/schemas/macros.json index 4ae7f2d..5a52524 100644 --- a/tap_zendesk/schemas/macros.json +++ b/tap_zendesk/schemas/macros.json @@ -54,7 +54,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "url": { "type": [ @@ -72,7 +73,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "active": { "type": [ diff --git a/tap_zendesk/schemas/organizations.json b/tap_zendesk/schemas/organizations.json index 1378f01..5f51fad 100644 --- a/tap_zendesk/schemas/organizations.json +++ b/tap_zendesk/schemas/organizations.json @@ -14,7 +14,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "tags": { "type": [ @@ -75,7 +76,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "name": { "type": [ diff --git a/tap_zendesk/schemas/ticket-audits.json b/tap_zendesk/schemas/ticket-audits.json index 1f0ab24..aca1aa8 100644 --- a/tap_zendesk/schemas/ticket-audits.json +++ b/tap_zendesk/schemas/ticket-audits.json @@ -190,8 +190,8 @@ "value": { "type": [ "null", - "string", - "array" + "array", + "string" ], "items": { "type": [ @@ -373,8 +373,8 @@ "previous_value": { "type": [ "null", - "string", - "array" + "array", + "string" ], "items": { "type": [ @@ -559,7 +559,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "ticket_id": { "type": [ diff --git a/tap_zendesk/schemas/ticket-fields.json b/tap_zendesk/schemas/ticket-fields.json index ff933b9..aee4efc 100644 --- a/tap_zendesk/schemas/ticket-fields.json +++ b/tap_zendesk/schemas/ticket-fields.json @@ -4,7 +4,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "title_in_portal": { "type": [ @@ -114,7 +115,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "tag": { "type": [ diff --git a/tap_zendesk/schemas/ticket-metrics.json b/tap_zendesk/schemas/ticket-metrics.json index f36d389..b3adee4 100644 --- a/tap_zendesk/schemas/ticket-metrics.json +++ b/tap_zendesk/schemas/ticket-metrics.json @@ -86,7 +86,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "first_resolution_time_in_minutes": { "type": [ @@ -138,7 +139,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "on_hold_time_in_minutes": { "type": [ @@ -196,7 +198,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "requester_wait_time_in_minutes": { "type": [ @@ -222,13 +225,15 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "updated_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, "url": { "type": [ @@ -240,25 +245,29 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "assigned_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, "solved_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, "assignee_updated_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" } }, "type": [ diff --git a/tap_zendesk/schemas/tickets.json b/tap_zendesk/schemas/tickets.json index 4032771..99e9b53 100644 --- a/tap_zendesk/schemas/tickets.json +++ b/tap_zendesk/schemas/tickets.json @@ -112,7 +112,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "raw_subject": { "type": [ @@ -130,7 +131,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "custom_fields": { "items": { @@ -141,20 +143,7 @@ "integer" ] }, - "value": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "string", - "boolean", - "array" - ] - } + "value": { } }, "type": [ "null", @@ -182,7 +171,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "followup_ids": { "items": { diff --git a/tap_zendesk/schemas/users.json b/tap_zendesk/schemas/users.json index 7967613..8ec92f6 100644 --- a/tap_zendesk/schemas/users.json +++ b/tap_zendesk/schemas/users.json @@ -92,7 +92,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "external_id": { "type": [ @@ -270,7 +271,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "suspended": { "type": [ @@ -301,7 +303,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "alias": { "type": [ diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index a0a716c..b6756e0 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -138,7 +138,9 @@ def sync(self, state): tickets = self.client.tickets.incremental(start_time=bookmark) for ticket in tickets: self.update_bookmark(state, ticket.updated_at) - yield ticket + ticket_dict = ticket.to_dict() + ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting + yield ticket_dict class TicketAudits(Stream): name = "ticket-audits" @@ -188,7 +190,7 @@ def sync(self, state): class Tags(Stream): name = "tags" replication_method = "FULL_TABLE" - key_properties = [] + key_properties = ["name"] def sync(self, state): # pylint: disable=unused-argument # NB: Setting page to force it to paginate all tags, instead of just the From 0d8ffd371c39ecf518bc50795affeb5989408360 Mon Sep 17 00:00:00 2001 From: Kyle Allan Date: Wed, 9 May 2018 11:03:31 -0400 Subject: [PATCH 2/3] minor cleanup; set valid-replication-keys metadata --- tap_zendesk/discover.py | 11 ----------- tap_zendesk/streams.py | 3 ++- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tap_zendesk/discover.py b/tap_zendesk/discover.py index 3442bcf..180dc71 100644 --- a/tap_zendesk/discover.py +++ b/tap_zendesk/discover.py @@ -6,14 +6,3 @@ def discover_streams(client): s = s(client) streams.append({'stream': s.name, 'tap_stream_id': s.name, 'schema': s.load_schema(), 'metadata': s.load_metadata()}) return streams - -# * Ticket Comments (P2) -# * Others from Audit (P2) - -# P2? -# Use "search" Endpoint; for things like ticket_metrics -# * zenpy_client.search("some query", type='ticket_metrics', sort_by='created_at', sort_order='desc') - -# P2? -# "Relationships" (?) -# tickets + ticket_fields = a ticket row diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index b6756e0..9a1a4cf 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -75,9 +75,10 @@ def load_metadata(self): mdata = metadata.write(mdata, (), 'table-key-properties', self.key_properties) mdata = metadata.write(mdata, (), 'forced-replication-method', self.replication_method) + mdata = metadata.write(mdata, (), 'valid-replication-keys', [self.replication_key]) for field_name in schema['properties'].keys(): - if field_name in KEY_PROPERTIES: + if field_name in KEY_PROPERTIES or field_name == self.replication_key: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic') else: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available') From f17388c49c7ddb37e0f8652b70e50963d82e085f Mon Sep 17 00:00:00 2001 From: Kyle Allan Date: Wed, 9 May 2018 15:53:51 -0400 Subject: [PATCH 3/3] only sync the correct ticket rows --- tap_zendesk/streams.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 9a1a4cf..1e2fd3a 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -138,6 +138,11 @@ def sync(self, state): bookmark = self.get_bookmark(state) tickets = self.client.tickets.incremental(start_time=bookmark) for ticket in tickets: + if utils.strptime_with_tz(ticket.updated_at) < bookmark: + # NB: Skip tickets that might show up because of Zendesk behavior: + # The Incremental Ticket Export endpoint also returns tickets that + # were updated for reasons not related to ticket events, such as a system update or a database backfill. + continue self.update_bookmark(state, ticket.updated_at) ticket_dict = ticket.to_dict() ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting