Several improvements

.pylintrc

@@ -0,0 +1,2 @@
+[MESSAGES CONTROL]
+disable=missing-docstring,too-few-public-methods,too-many-arguments,logging-format-interpolation

README.md

@@ -19,6 +19,29 @@ This tap:
 - Outputs the schema for each resource
 - Incrementally pulls data based on the input state
 
+## Configuration
+
+Create a `config.json` file that looks like this:
+
+```json
+{
+    "start_date": "2010-01-01",
+    "access_token": "your-access-token",
+    "user_agent": "tap-zendesk-chat <email@my_domain.com>"
+}
+```
+
+Overview over the possible config properties:
+
+| Config property             | Required / Default value | Description
+| --------------------------- | ------------------------ | -----------
+| `start_date`                | Yes                      | For streams with replication method INCREMENTAL the start date time to be used
+| `access_token`              | Yes                      | Your Zen Desk Chat access token
+| `user_agent`                | No                       | User agent to be used for HTTP requests
+| `agents_page_limit`         | No, default `"100"`      | page limit for stream `agents`
+| `chat_search_interval_days` | No, default: `"14"`      | the interval in days for stream `chats`
+| `chats_full_sync_days`      | No                       | See section "Chats Full Re-syncs" below
+
 ## Quick Start
 
 1. Install
@@ -43,16 +66,8 @@ Zopim account.
 
 3. Create the Config File
 
-Create a JSON file called `config.json` containing the access token and a
-`start_date`, which specifies the date at which the tap will begin pulling data
-(for those resources that support this).
-
-```json
-{
-    "start_date": "2010-01-01",
-    "access_token": "your-access-token"
-}
-```
+Create a JSON file called `config.json` from the `sample_config.json` file in this
+repository.
 
 4. Run the Tap in Discovery Mode
 
@@ -94,4 +109,5 @@ tap's "state."
 
 ---
 
-Copyright &copy; 2017 Stitch
+Copyright &copy; 2017 Stitch<br/>
+Copyright &copy; 2020 Horze International GmbH

sample_config.json

@@ -0,0 +1,5 @@
+{
+    "start_date": "2010-01-01",
+    "access_token": "your-access-token",
+    "user_agent": "tap-zendesk-chat <email@my_domain.com>"
+}

setup.py

@@ -9,9 +9,7 @@
       classifiers=["Programming Language :: Python :: 3 :: Only"],
       py_modules=["tap_zendesk_chat"],
       install_requires=[
-          "python-dateutil==2.6.0",  # because of singer-python issue
-          "pendulum==1.2.0",  # because of singer-python issue
-          "singer-python==5.0.3",
+          "singer-python==5.9.1",
           "requests==2.20.0",
       ],
       entry_points="""

tap_zendesk_chat/__init__.py

@@ -5,8 +5,10 @@
 from singer.catalog import Catalog, CatalogEntry, Schema
 from requests.exceptions import HTTPError
 from . import streams as streams_
+from .streams import STREAM_OBJECTS
 from .context import Context
 from .http import Client
+from .sync import sync
 
 REQUIRED_CONFIG_KEYS = ["start_date", "access_token"]
 LOGGER = singer.get_logger()
@@ -30,14 +32,14 @@ def load_schema(tap_stream_id):
 
 def ensure_credentials_are_authorized(client):
     # The request will throw an exception if the credentials are not authorized
-    client.request(streams_.DEPARTMENTS.tap_stream_id)
+    client.request(STREAM_OBJECTS['departments'].tap_stream_id)
 
 
 def is_account_endpoint_authorized(client):
     # The account endpoint is restricted to zopim accounts, meaning integrated
     # Zendesk accounts will get a 403 for this endpoint.
     try:
-        client.request(streams_.ACCOUNT.tap_stream_id)
+        client.request(STREAM_OBJECTS['account'].tap_stream_id)
     except HTTPError as e:
         if e.response.status_code == 403:
             LOGGER.info(
@@ -46,70 +48,43 @@ def is_account_endpoint_authorized(client):
                 "from discovery."
             )
             return False
-        else:
-            raise
+        raise
     return True
 
 
 def discover(config):
     client = Client(config)
     ensure_credentials_are_authorized(client)
     include_account_stream = is_account_endpoint_authorized(client)
-    catalog = Catalog([])
-    for stream in streams_.all_streams:
+    streams = []
+    for _, stream in STREAM_OBJECTS.items():
         if (not include_account_stream
-            and stream.tap_stream_id == streams_.ACCOUNT.tap_stream_id):
+            and stream.tap_stream_id == STREAM_OBJECTS['account'].tap_stream_id):
             continue
         raw_schema = load_schema(stream.tap_stream_id)
-        mdata = build_metadata(raw_schema)
         schema = Schema.from_dict(raw_schema)
-        catalog.streams.append(CatalogEntry(
+        streams.append(CatalogEntry(
             stream=stream.tap_stream_id,
             tap_stream_id=stream.tap_stream_id,
             key_properties=stream.pk_fields,
             schema=schema,
-            metadata=metadata.to_list(mdata)
+            metadata=metadata.get_standard_metadata(
+                schema=raw_schema,
+                schema_name=stream.tap_stream_id,
+                key_properties=stream.pk_fields,
+                valid_replication_keys=stream.replication_keys,
+                replication_method=stream.replication_method)
         ))
-    return catalog
-
-def build_metadata(raw_schema):
-    mdata = metadata.new()
-    for prop in raw_schema['properties'].keys():
-        metadata.write(mdata, ('properties', prop), 'inclusion', 'automatic')
-    return mdata
-
-
-def output_schema(stream):
-    schema = load_schema(stream.tap_stream_id)
-    singer.write_schema(stream.tap_stream_id, schema, stream.pk_fields)
-
-
-def is_selected(stream):
-    mdata = metadata.to_map(stream.metadata)
-    return metadata.get(mdata, (), 'selected')
-
-def sync(ctx):
-    currently_syncing = ctx.state.get("currently_syncing")
-    start_idx = streams_.all_stream_ids.index(currently_syncing) \
-        if currently_syncing else 0
-    stream_ids_to_sync = [cs.tap_stream_id for cs in ctx.catalog.streams
-                          if is_selected(cs)]
-    streams = [s for s in streams_.all_streams[start_idx:]
-               if s.tap_stream_id in stream_ids_to_sync]
-    for stream in streams:
-        ctx.state["currently_syncing"] = stream.tap_stream_id
-        output_schema(stream)
-        ctx.write_state()
-        stream.sync(ctx)
-    ctx.state["currently_syncing"] = None
-    ctx.write_state()
+    return Catalog(streams)
 
 
 def main_impl():
     args = utils.parse_args(REQUIRED_CONFIG_KEYS)
     if args.discover:
         discover(args.config).dump()
-        print()
+    elif args.catalog:
+        ctx = Context(args.config, args.state, args.catalog)
+        sync(ctx)
     else:
         catalog = Catalog.from_dict(args.properties) \
             if args.properties else discover(args.config)

tap_zendesk_chat/context.py

@@ -1,16 +1,16 @@
-from datetime import datetime
-from .http import Client
+from datetime import datetime, timezone
 import singer
-from datetime import datetime
+
+from .http import Client
 
 
-class Context(object):
+class Context:
     def __init__(self, config, state, catalog):
         self.config = config
         self.state = state
         self.catalog = catalog
         self.client = Client(config)
-        self.now = datetime.utcnow()
+        self.now = datetime.utcnow().replace(tzinfo=timezone.utc)
 
     @property
     def bookmarks(self):

tap_zendesk_chat/http.py

@@ -9,7 +9,7 @@ class RateLimitException(Exception):
     pass
 
 
-class Client(object):
+class Client:
     def __init__(self, config):
         # self.session = requests.Session()
         self.access_token = config["access_token"]