Twitter integration

diffengine/__init__.py

@@ -28,14 +28,16 @@
 
 from peewee import *
 from playhouse.migrate import SqliteMigrator, migrate
-from datetime import datetime, timedelta
+from datetime import datetime
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options as ChromeOptions
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
 from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
 from envyaml import EnvYAML
 
-from diffengine.exceptions import UnknownWebdriverError
+from exceptions.webdriver import UnknownWebdriverError
+from exceptions.twitter import ConfigNotFoundError, TwitterError
+from diffengine.twitter import TwitterHandler
 
 home = None
 config = {}
@@ -97,6 +99,7 @@ class Entry(BaseModel):
     url = CharField()
     created = DateTimeField(default=datetime.utcnow)
     checked = DateTimeField(default=datetime.utcnow)
+    tweet_status_id_str = CharField(null=True)
 
     @property
     def feeds(self):
@@ -195,7 +198,12 @@ def get_latest(self):
                 logging.debug("found new version %s", old.entry.url)
                 diff = Diff.create(old=old, new=new)
                 if not diff.generate():
-                    logging.warn("html diff showed no changes: %s", self.url)
+                    logging.warn(
+                        "html diff showed no changes between versions #%s and #%s: %s",
+                        old.id,
+                        new.id,
+                        self.url,
+                    )
                     new.delete()
                     new = None
             else:
@@ -222,6 +230,7 @@ class EntryVersion(BaseModel):
     created = DateTimeField(default=datetime.utcnow)
     archive_url = CharField(null=True)
     entry = ForeignKeyField(Entry, backref="versions")
+    tweet_status_id_str = CharField(null=True)
 
     @property
     def diff(self):
@@ -306,6 +315,7 @@ def snap(url):
             snap(self.old.archive_url), snap(self.new.archive_url), self.old.url
         )
 
+    # TODO: configurable option for deleting the diffs after some time (1 week?)
     def generate(self):
         if self._generate_diff_html():
             self._generate_diff_images()
@@ -489,41 +499,6 @@ def setup_browser(engine="geckodriver", executable_path=None, binary_location=""
         return geckodriver_browser()
 
 
-def tweet_diff(diff, token):
-    if "twitter" not in config:
-        logging.debug("twitter not configured")
-        return
-    elif not token:
-        logging.debug("access token/secret not set up for feed")
-        return
-    elif diff.tweeted:
-        logging.warn("diff %s has already been tweeted", diff.id)
-        return
-    elif not (diff.old.archive_url and diff.new.archive_url):
-        logging.warn("not tweeting without archive urls")
-        return
-
-    t = config["twitter"]
-    auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"])
-    auth.secure = True
-    auth.set_access_token(token["access_token"], token["access_token_secret"])
-    twitter = tweepy.API(auth)
-
-    status = diff.new.title
-    if len(status) >= 225:
-        status = status[0:225] + "…"
-
-    status += " " + diff.url
-
-    try:
-        twitter.update_with_media(diff.thumbnail_path, status)
-        diff.tweeted = datetime.utcnow()
-        logging.info("tweeted %s", status)
-        diff.save()
-    except Exception as e:
-        logging.error("unable to tweet: %s", e)
-
-
 def init(new_home, prompt=True):
     global home, browser
     home = new_home
@@ -550,6 +525,18 @@ def main():
     start_time = datetime.utcnow()
     logging.info("starting up with home=%s", home)
 
+    try:
+        twitter_config = config.get("twitter")
+        twitter_handler = TwitterHandler(
+            twitter_config["consumer_key"], twitter_config["consumer_secret"]
+        )
+    except ConfigNotFoundError as e:
+        twitter_handler = None
+        logging.warning("error when creating Twitter Handler. Reason", str(e))
+    except KeyError as e:
+        twitter_handler = None
+        logging.warning("the twitter keys are not present in config. Reason", str(e))
+
     checked = skipped = new = 0
 
     for f in config.get("feeds", []):
@@ -562,19 +549,10 @@ def main():
 
         # get latest content for each entry
         for entry in feed.entries:
-            if not entry.stale:
-                skipped += 1
-                continue
-            checked += 1
-            try:
-                version = entry.get_latest()
-            except Exception as e:
-                logging.error("unable to get latest", e)
-                continue
-            if version:
-                new += 1
-            if version and version.diff and "twitter" in f:
-                tweet_diff(version.diff, f["twitter"])
+            result = process_entry(entry, f["twitter"], twitter_handler)
+            skipped += result["skipped"]
+            checked += result["checked"]
+            new += result["new"]
 
     elapsed = datetime.utcnow() - start_time
     logging.info(
@@ -588,6 +566,28 @@ def main():
     browser.quit()
 
 
+def process_entry(entry, token=None, twitter=None):
+    result = {"skipped": 0, "checked": 0, "new": 0}
+    if not entry.stale:
+        result["skipped"] = 1
+    else:
+        result["checked"] = 1
+        try:
+            version = entry.get_latest()
+            if version:
+                result["new"] = 1
+                if version.diff and token is not None:
+                    try:
+                        twitter.tweet_diff(version.diff, token)
+                    except TwitterError as e:
+                        logging.warning("error occurred while trying to tweet", e)
+                    except Exception as e:
+                        logging.error("unknown error when tweeting diff", e)
+        except Exception as e:
+            logging.error("unable to get latest", e)
+    return result
+
+
 def _dt(d):
     return d.strftime("%Y-%m-%d %H:%M:%S")
 

diffengine/twitter.py

@@ -0,0 +1,99 @@
+import logging
+import tweepy
+
+from datetime import datetime
+
+from exceptions.twitter import (
+    AlreadyTweetedError,
+    ConfigNotFoundError,
+    TokenNotFoundError,
+    AchiveUrlNotFoundError,
+    UpdateStatusError,
+)
+
+
+class TwitterHandler:
+    consumer_key = None
+    consumer_secret = None
+
+    def __init__(self, consumer_key, consumer_secret):
+        if not consumer_key or not consumer_secret:
+            raise ConfigNotFoundError()
+
+        self.consumer_key = consumer_key
+        self.consumer_secret = consumer_secret
+
+        auth = tweepy.OAuthHandler(self.consumer_key, self.consumer_secret)
+        auth.secure = True
+        self.auth = auth
+
+    def api(self, token):
+        self.auth.set_access_token(token["access_token"], token["access_token_secret"])
+        return tweepy.API(self.auth)
+
+    def build_text(self, diff):
+        text = diff.new.title
+        if len(text) >= 225:
+            text = text[0:225] + "…"
+        text += " " + diff.url
+        return text
+
+    def create_thread(self, entry, first_version, token):
+        try:
+            twitter = self.api(token)
+            status = twitter.update_status(entry.url)
+            entry.tweet_status_id_str = status.id_str
+            entry.save()
+
+            # Save the entry status_id inside the first entryVersion
+            first_version.tweet_status_id_str = status.id_str
+            first_version.save()
+            return status.id_str
+        except Exception as e:
+            raise UpdateStatusError(entry)
+
+    def tweet_diff(self, diff, token=None):
+        if not token:
+            raise TokenNotFoundError()
+        elif diff.tweeted:
+            raise AlreadyTweetedError(diff.id)
+        elif not (diff.old.archive_url and diff.new.archive_url):
+            raise AchiveUrlNotFoundError()
+
+        twitter = self.api(token)
+        text = self.build_text(diff)
+
+        # Check if the thread exists
+        thread_status_id_str = None
+        if diff.old.entry.tweet_status_id_str is None:
+            try:
+                thread_status_id_str = self.create_thread(
+                    diff.old.entry, diff.old, token
+                )
+                logging.info(
+                    "created thread https://twitter.com/%s/status/%s"
+                    % (self.auth.get_username(), thread_status_id_str)
+                )
+            except UpdateStatusError as e:
+                logging.error(str(e))
+        else:
+            thread_status_id_str = diff.old.tweet_status_id_str
+
+        try:
+            status = twitter.update_with_media(
+                diff.thumbnail_path,
+                status=text,
+                in_reply_to_status_id=thread_status_id_str,
+            )
+            logging.info(
+                "tweeted diff https://twitter.com/%s/status/%s"
+                % (self.auth.get_username(), status.id_str)
+            )
+            # Save the tweet status id inside the new version
+            diff.new.tweet_status_id_str = status.id_str
+            diff.new.save()
+            # And save that the diff has been tweeted
+            diff.tweeted = datetime.utcnow()
+            diff.save()
+        except Exception as e:
+            logging.error("unable to tweet: %s", e)

exceptions/twitter.py

@@ -0,0 +1,34 @@
+class TwitterError(RuntimeError):
+    pass
+
+
+class ConfigNotFoundError(TwitterError):
+    """Exception raised if the Twitter instance has not the required key and secret"""
+
+    def __init__(self):
+        self.message = "consumer key/secret not set up for feed."
+
+
+class TokenNotFoundError(TwitterError):
+    """Exception raised if no token is preset"""
+
+    def __init__(self):
+        self.message = "access token/secret not set up for feed"
+
+
+class AlreadyTweetedError(TwitterError):
+    def __init__(self, diff_id):
+        self.message = "diff %s has already been tweeted" % diff_id
+
+
+class AchiveUrlNotFoundError(TwitterError):
+    def __init__(self):
+        self.message = "not tweeting without archive urls"
+
+
+class UpdateStatusError(TwitterError):
+    def __init__(self, entry):
+        self.message = "could not create thread on entry id %s, url %s" % (
+            entry.id,
+            entry.url,
+        )