Skip to content

Commit

Permalink
Merge pull request #7 from singer-io/fix/data-diff
Browse files Browse the repository at this point in the history 
Schema Updates and various fixes
  • Loading branch information
@KAllan357
KAllan357 authored May 9, 2018
2 parents 977b959 + f17388c commit 875ad12
Show file tree
Hide file tree
Showing 11 changed files with 68 additions and 58 deletions.
11 changes: 0 additions & 11 deletions tap_zendesk/discover.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,3 @@ def discover_streams(client):
s = s(client)
streams.append({'stream': s.name, 'tap_stream_id': s.name, 'schema': s.load_schema(), 'metadata': s.load_metadata()})
return streams

# * Ticket Comments (P2)
# * Others from Audit (P2)

# P2?
# Use "search" Endpoint; for things like ticket_metrics
# * zenpy_client.search("some query", type='ticket_metrics', sort_by='created_at', sort_order='desc')

# P2?
# "Relationships" (?)
# tickets + ticket_fields = a ticket row
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/group-memberships.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"group_id": {
"type": [
Expand All @@ -34,7 +35,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"id": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/groups.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"url": {
"type": [
Expand All @@ -26,7 +27,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"deleted": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/macros.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"url": {
"type": [
Expand All @@ -72,7 +73,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"active": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/organizations.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"tags": {
"type": [
Expand Down Expand Up @@ -75,7 +76,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"name": {
"type": [
Expand Down
11 changes: 6 additions & 5 deletions tap_zendesk/schemas/ticket-audits.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,8 @@
"value": {
"type": [
"null",
"string",
"array"
"array",
"string"
],
"items": {
"type": [
Expand Down Expand Up @@ -373,8 +373,8 @@
"previous_value": {
"type": [
"null",
"string",
"array"
"array",
"string"
],
"items": {
"type": [
Expand Down Expand Up @@ -559,7 +559,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"ticket_id": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/ticket-fields.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"title_in_portal": {
"type": [
Expand Down Expand Up @@ -114,7 +115,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"tag": {
"type": [
Expand Down
27 changes: 18 additions & 9 deletions tap_zendesk/schemas/ticket-metrics.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"first_resolution_time_in_minutes": {
"type": [
Expand Down Expand Up @@ -138,7 +139,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"on_hold_time_in_minutes": {
"type": [
Expand Down Expand Up @@ -196,7 +198,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"requester_wait_time_in_minutes": {
"type": [
Expand All @@ -222,13 +225,15 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"updated_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"url": {
"type": [
Expand All @@ -240,25 +245,29 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"assigned_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"solved_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"assignee_updated_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
}
},
"type": [
Expand Down
24 changes: 7 additions & 17 deletions tap_zendesk/schemas/tickets.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"raw_subject": {
"type": [
Expand All @@ -130,7 +131,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"custom_fields": {
"items": {
Expand All @@ -141,20 +143,7 @@
"integer"
]
},
"value": {
"items": {
"type": [
"null",
"string"
]
},
"type": [
"null",
"string",
"boolean",
"array"
]
}
"value": { }
},
"type": [
"null",
Expand Down Expand Up @@ -182,7 +171,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"followup_ids": {
"items": {
Expand Down
9 changes: 6 additions & 3 deletions tap_zendesk/schemas/users.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"external_id": {
"type": [
Expand Down Expand Up @@ -270,7 +271,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"suspended": {
"type": [
Expand Down Expand Up @@ -301,7 +303,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"alias": {
"type": [
Expand Down
14 changes: 11 additions & 3 deletions tap_zendesk/streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ def load_metadata(self):

mdata = metadata.write(mdata, (), 'table-key-properties', self.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', self.replication_method)
mdata = metadata.write(mdata, (), 'valid-replication-keys', [self.replication_key])

for field_name in schema['properties'].keys():
if field_name in KEY_PROPERTIES:
if field_name in KEY_PROPERTIES or field_name == self.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
Expand Down Expand Up @@ -137,8 +138,15 @@ def sync(self, state):
bookmark = self.get_bookmark(state)
tickets = self.client.tickets.incremental(start_time=bookmark)
for ticket in tickets:
if utils.strptime_with_tz(ticket.updated_at) < bookmark:
# NB: Skip tickets that might show up because of Zendesk behavior:
# The Incremental Ticket Export endpoint also returns tickets that
# were updated for reasons not related to ticket events, such as a system update or a database backfill.
continue
self.update_bookmark(state, ticket.updated_at)
yield ticket
ticket_dict = ticket.to_dict()
ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting
yield ticket_dict

class TicketAudits(Stream):
name = "ticket-audits"
Expand Down Expand Up @@ -188,7 +196,7 @@ def sync(self, state):
class Tags(Stream):
name = "tags"
replication_method = "FULL_TABLE"
key_properties = []
key_properties = ["name"]

def sync(self, state): # pylint: disable=unused-argument
# NB: Setting page to force it to paginate all tags, instead of just the
Expand Down

0 comments on commit 875ad12

Please sign in to comment.