From 33a760b0cdddf03ea84bf49b528da5a27130e896 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 01:16:01 -0300 Subject: [PATCH 01/23] Twitter thread if at least one diff. Need to create classes to be able to test properly --- diffengine/__init__.py | 72 ++++++++++-------------------- diffengine/tweet_utils.py | 94 +++++++++++++++++++++++++++++++++++++++ test_diffengine.py | 93 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 207 insertions(+), 52 deletions(-) create mode 100644 diffengine/tweet_utils.py diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 3970697..b74fe92 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -28,7 +28,7 @@ from peewee import * from playhouse.migrate import SqliteMigrator, migrate -from datetime import datetime, timedelta +from datetime import datetime from selenium import webdriver from selenium.webdriver.chrome.options import Options as ChromeOptions from selenium.webdriver.firefox.options import Options as FirefoxOptions @@ -36,6 +36,7 @@ from envyaml import EnvYAML from diffengine.exceptions import UnknownWebdriverError +from diffengine.tweet_utils import tweet_diff, tweet_thread home = None config = {} @@ -489,41 +490,6 @@ def setup_browser(engine="geckodriver", executable_path=None, binary_location="" return geckodriver_browser() -def tweet_diff(diff, token): - if "twitter" not in config: - logging.debug("twitter not configured") - return - elif not token: - logging.debug("access token/secret not set up for feed") - return - elif diff.tweeted: - logging.warn("diff %s has already been tweeted", diff.id) - return - elif not (diff.old.archive_url and diff.new.archive_url): - logging.warn("not tweeting without archive urls") - return - - t = config["twitter"] - auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) - auth.secure = True - auth.set_access_token(token["access_token"], token["access_token_secret"]) - twitter = tweepy.API(auth) - - status = diff.new.title - if len(status) >= 225: - status = status[0:225] + "…" - - status += " " + diff.url - - try: - twitter.update_with_media(diff.thumbnail_path, status) - diff.tweeted = datetime.utcnow() - logging.info("tweeted %s", status) - diff.save() - except Exception as e: - logging.error("unable to tweet: %s", e) - - def init(new_home, prompt=True): global home, browser home = new_home @@ -562,19 +528,10 @@ def main(): # get latest content for each entry for entry in feed.entries: - if not entry.stale: - skipped += 1 - continue - checked += 1 - try: - version = entry.get_latest() - except Exception as e: - logging.error("unable to get latest", e) - continue - if version: - new += 1 - if version and version.diff and "twitter" in f: - tweet_diff(version.diff, f["twitter"]) + result = process_entry(entry, f["twitter"]) + skipped += result["skipped"] + checked += result["checked"] + new += result["new"] elapsed = datetime.utcnow() - start_time logging.info( @@ -588,6 +545,23 @@ def main(): browser.quit() +def process_entry(entry, token=None): + result = {"skipped": 0, "checked": 0, "new": 0} + if not entry.stale: + result["skipped"] = 1 + else: + result["checked"] = 1 + try: + version = entry.get_latest() + result["new"] = 1 + if version.diff and token is not None: + tweet_diff(version.diff, token, config) + except Exception as e: + logging.error("unable to get latest", e) + return result + return result + + def _dt(d): return d.strftime("%Y-%m-%d %H:%M:%S") diff --git a/diffengine/tweet_utils.py b/diffengine/tweet_utils.py new file mode 100644 index 0000000..24f713e --- /dev/null +++ b/diffengine/tweet_utils.py @@ -0,0 +1,94 @@ +import logging +import tweepy + +from datetime import datetime + + +def tweet_thread(entry, first_version, token, config): + if config.get("twitter", None) is None: + logging.debug("twitter not configured") + return + elif not token: + logging.debug("access token/secret not set up for feed") + return + elif entry.tweet_status_id_str: + logging.warning("entry %s has already been tweeted", entry.id) + return + + t = config["twitter"] + auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) + auth.secure = True + auth.set_access_token(token["access_token"], token["access_token_secret"]) + twitter = tweepy.API(auth) + + status = twitter.update_status(entry.url) + entry.tweet_status_id_str = status.id_str + entry.save() + + # Save the entry status_id inside the first entryVersion + first_version.tweet_status_id_str = status.id_str + first_version.save() + return status.id_str + + +def tweet_diff(diff, token, config): + if config.get("twitter", None) is None: + logging.debug("twitter not configured") + return + elif not token: + logging.debug("access token/secret not set up for feed") + return + elif diff.tweeted: + logging.warning("diff %s has already been tweeted", diff.id) + return + elif not (diff.old.archive_url and diff.new.archive_url): + logging.warning("not tweeting without archive urls") + return + + t = config["twitter"] + auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) + auth.secure = True + auth.set_access_token(token["access_token"], token["access_token_secret"]) + twitter = tweepy.API(auth) + + text = build_text(diff) + + # Check if the thread exists + thread_status_id_str = None + if diff.old.entry.tweet_status_id_str is None: + try: + thread_status_id_str = tweet_thread(diff.old.entry, diff.old, token, config) + logging.info( + "created thread https://twitter/%s/status/%s" + % (auth.get_username(), thread_status_id_str) + ) + except Exception as e: + logging.error("could not create thread on entry %s" % diff.old.entry.url, e) + else: + thread_status_id_str = diff.old.tweet_status_id_str + + try: + status = twitter.update_with_media( + diff.thumbnail_path, status=text, in_reply_to_status_id=thread_status_id_str + ) + logging.info( + "tweeted diff https://twitter.com/%s/status/%s" + % (auth.get_username(), status.id_str) + ) + # Save the tweet status id inside the new version + diff.new.tweet_status_id_str = status.id_str + diff.new.save() + # And save that the diff has been tweeted + diff.tweeted = datetime.utcnow() + diff.save() + except Exception as e: + logging.error("unable to tweet: %s", e) + + +def build_text(diff): + text = diff.new.title + if len(text) >= 225: + text = text[0:225] + "…" + text += " " + diff.url + + return text diff --git a/test_diffengine.py b/test_diffengine.py index b6e8689..c7977fd 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -1,17 +1,36 @@ import os import re + +import yaml +from selenium import webdriver + import setup import pytest import shutil -from diffengine import * +from unittest.mock import patch +from unittest.mock import MagicMock +from unittest.mock import PropertyMock + +from diffengine import ( + init, + Feed, + EntryVersion, + Entry, + FeedEntry, + home_path, + load_config, + setup_browser, + UnknownWebdriverError, + process_entry, + UA, +) if os.path.isdir("test"): shutil.rmtree("test") # set things up but disable prompting for initial feed -test_home = "test" -init(test_home, prompt=False) +init("test", prompt=False) # the sequence of these tests is significant @@ -183,3 +202,71 @@ def test_webdriver_is_chromedriver(): # create config.yaml that will be read browser = setup_browser("chromedriver") assert isinstance(browser, webdriver.Chrome) == True + + +def test_entry_stale(): + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=False) + result = process_entry(entry) + assert result["skipped"] == 1 + + +def test_entry_stale(): + # Mock the entry and the stale property result + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=False) + result = process_entry(entry) + assert result["skipped"] == 1 + + +def test_twitter_do_nothing_if_entry_has_no_diff(): + # Mock that returns no diff + tweet_diff = MagicMock(name="tweet_diff") + version = MagicMock() + type(version).diff = PropertyMock(return_value=None) + + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=version) + + result = process_entry(entry, None) + assert result["checked"] == 1 + assert result["new"] == 1 + assert entry.get_latest.called + assert not tweet_diff.called + + +def test_twitter_do_nothing_if_feed_has_no_token(): + # Mock that returns no diff but it has no token + tweet_diff = MagicMock(name="tweet_diff") + version = MagicMock() + type(version).diff = PropertyMock(return_value=None) + + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=version) + + result = process_entry(entry, None) + assert result["checked"] == 1 + assert result["new"] == 1 + assert entry.get_latest.called + assert not tweet_diff.called + + +def test_twitter_do_tweet_diff_if_entry_has_diff(): + # Mock that returns a diff + tweet_diff = MagicMock(name="tweet_diff") + version = MagicMock() + type(version).diff = PropertyMock(return_value=MagicMock()) + + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=version) + + token = {"access_token": "test", "access_token_secret": "test"} + result = process_entry(entry, token) + + assert result["checked"] == 1 + assert result["new"] == 1 + assert entry.get_latest.called + assert tweet_diff.called From cdf403b1aa44540c550036cda48d86c3f9be05e3 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 09:42:56 -0300 Subject: [PATCH 02/23] Twitter tests! --- diffengine/__init__.py | 10 ++-- diffengine/tweet_utils.py | 94 ------------------------------- diffengine/twitter.py | 108 ++++++++++++++++++++++++++++++++++++ test_diffengine.py | 114 +++++++++++++++++++++----------------- 4 files changed, 176 insertions(+), 150 deletions(-) delete mode 100644 diffengine/tweet_utils.py create mode 100644 diffengine/twitter.py diff --git a/diffengine/__init__.py b/diffengine/__init__.py index b74fe92..27bcf11 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -36,7 +36,7 @@ from envyaml import EnvYAML from diffengine.exceptions import UnknownWebdriverError -from diffengine.tweet_utils import tweet_diff, tweet_thread +from diffengine.twitter import Twitter home = None config = {} @@ -526,9 +526,11 @@ def main(): # get latest feed entries feed.get_latest() + twitter = Twitter(config) + # get latest content for each entry for entry in feed.entries: - result = process_entry(entry, f["twitter"]) + result = process_entry(entry, f["twitter"], twitter) skipped += result["skipped"] checked += result["checked"] new += result["new"] @@ -545,7 +547,7 @@ def main(): browser.quit() -def process_entry(entry, token=None): +def process_entry(entry, token=None, twitter=None): result = {"skipped": 0, "checked": 0, "new": 0} if not entry.stale: result["skipped"] = 1 @@ -555,7 +557,7 @@ def process_entry(entry, token=None): version = entry.get_latest() result["new"] = 1 if version.diff and token is not None: - tweet_diff(version.diff, token, config) + twitter.tweet_diff(version.diff, token) except Exception as e: logging.error("unable to get latest", e) return result diff --git a/diffengine/tweet_utils.py b/diffengine/tweet_utils.py deleted file mode 100644 index 24f713e..0000000 --- a/diffengine/tweet_utils.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -import tweepy - -from datetime import datetime - - -def tweet_thread(entry, first_version, token, config): - if config.get("twitter", None) is None: - logging.debug("twitter not configured") - return - elif not token: - logging.debug("access token/secret not set up for feed") - return - elif entry.tweet_status_id_str: - logging.warning("entry %s has already been tweeted", entry.id) - return - - t = config["twitter"] - auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) - auth.secure = True - auth.set_access_token(token["access_token"], token["access_token_secret"]) - twitter = tweepy.API(auth) - - status = twitter.update_status(entry.url) - entry.tweet_status_id_str = status.id_str - entry.save() - - # Save the entry status_id inside the first entryVersion - first_version.tweet_status_id_str = status.id_str - first_version.save() - return status.id_str - - -def tweet_diff(diff, token, config): - if config.get("twitter", None) is None: - logging.debug("twitter not configured") - return - elif not token: - logging.debug("access token/secret not set up for feed") - return - elif diff.tweeted: - logging.warning("diff %s has already been tweeted", diff.id) - return - elif not (diff.old.archive_url and diff.new.archive_url): - logging.warning("not tweeting without archive urls") - return - - t = config["twitter"] - auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) - auth.secure = True - auth.set_access_token(token["access_token"], token["access_token_secret"]) - twitter = tweepy.API(auth) - - text = build_text(diff) - - # Check if the thread exists - thread_status_id_str = None - if diff.old.entry.tweet_status_id_str is None: - try: - thread_status_id_str = tweet_thread(diff.old.entry, diff.old, token, config) - logging.info( - "created thread https://twitter/%s/status/%s" - % (auth.get_username(), thread_status_id_str) - ) - except Exception as e: - logging.error("could not create thread on entry %s" % diff.old.entry.url, e) - else: - thread_status_id_str = diff.old.tweet_status_id_str - - try: - status = twitter.update_with_media( - diff.thumbnail_path, status=text, in_reply_to_status_id=thread_status_id_str - ) - logging.info( - "tweeted diff https://twitter.com/%s/status/%s" - % (auth.get_username(), status.id_str) - ) - # Save the tweet status id inside the new version - diff.new.tweet_status_id_str = status.id_str - diff.new.save() - # And save that the diff has been tweeted - diff.tweeted = datetime.utcnow() - diff.save() - except Exception as e: - logging.error("unable to tweet: %s", e) - - -def build_text(diff): - text = diff.new.title - if len(text) >= 225: - text = text[0:225] + "…" - text += " " + diff.url - - return text diff --git a/diffengine/twitter.py b/diffengine/twitter.py new file mode 100644 index 0000000..dcfb34e --- /dev/null +++ b/diffengine/twitter.py @@ -0,0 +1,108 @@ +import logging +import tweepy + +from datetime import datetime + + +def build_text(diff): + text = diff.new.title + if len(text) >= 225: + text = text[0:225] + "…" + text += " " + diff.url + + return text + + +class Twitter: + consumer_key = None + consumer_secret = None + + def __init__(self, config): + twitter_config = config.get("twitter", None) + if twitter_config is None: + logging.debug("twitter not configured") + return + elif ( + not twitter_config["consumer_key"] or not twitter_config["consumer_secret"] + ): + logging.debug("consumer key/secret not set up for feed") + return + + self.consumer_key = twitter_config["access_token"] + self.consumer_secret = twitter_config["access_secret"] + auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) + auth.secure = True + self.auth = auth + + def api(self, token): + self.auth.set_access_token(token["access_token"], token["access_token_secret"]) + return tweepy.API(self.auth) + + def tweet_thread(self, entry, first_version, token): + if not token: + logging.debug("access token/secret not set up for feed") + return + elif entry.tweet_status_id_str: + logging.warning("entry %s has already been tweeted", entry.id) + return + + twitter = self.api(token) + status = twitter.update_status(entry.url) + entry.tweet_status_id_str = status.id_str + entry.save() + + # Save the entry status_id inside the first entryVersion + first_version.tweet_status_id_str = status.id_str + first_version.save() + return status.id_str + + def tweet_diff(self, diff, token): + if not token: + logging.debug("access token/secret not set up for feed") + return + elif diff.tweeted: + logging.warning("diff %s has already been tweeted", diff.id) + return + elif not (diff.old.archive_url and diff.new.archive_url): + logging.warning("not tweeting without archive urls") + return + + twitter = self.api(token) + text = build_text(diff) + + # Check if the thread exists + thread_status_id_str = None + if diff.old.entry.tweet_status_id_str is None: + try: + thread_status_id_str = self.tweet_thread( + diff.old.entry, diff.old, token + ) + logging.info( + "created thread https://twitter/%s/status/%s" + % (self.auth.get_username(), thread_status_id_str) + ) + except Exception as e: + logging.error( + "could not create thread on entry %s" % diff.old.entry.url, e + ) + else: + thread_status_id_str = diff.old.tweet_status_id_str + + try: + status = twitter.update_with_media( + diff.thumbnail_path, + status=text, + in_reply_to_status_id=thread_status_id_str, + ) + logging.info( + "tweeted diff https://twitter.com/%s/status/%s" + % (self.auth.get_username(), status.id_str) + ) + # Save the tweet status id inside the new version + diff.new.tweet_status_id_str = status.id_str + diff.new.save() + # And save that the diff has been tweeted + diff.tweeted = datetime.utcnow() + diff.save() + except Exception as e: + logging.error("unable to tweet: %s", e) diff --git a/test_diffengine.py b/test_diffengine.py index c7977fd..b240231 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -8,7 +8,7 @@ import pytest import shutil -from unittest.mock import patch +from unittest import TestCase from unittest.mock import MagicMock from unittest.mock import PropertyMock @@ -204,69 +204,79 @@ def test_webdriver_is_chromedriver(): assert isinstance(browser, webdriver.Chrome) == True -def test_entry_stale(): - entry = MagicMock() - type(entry).stale = PropertyMock(return_value=False) - result = process_entry(entry) - assert result["skipped"] == 1 +class EntryTest(TestCase): + def test_stale_is_skipped(self): + # Prepare + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=False) + # Test + result = process_entry(entry) -def test_entry_stale(): - # Mock the entry and the stale property result - entry = MagicMock() - type(entry).stale = PropertyMock(return_value=False) - result = process_entry(entry) - assert result["skipped"] == 1 + # Assert + assert result["skipped"] == 1 + def test_do_not_tweet_if_entry_has_no_diff(self): + # Prepare + twitter = MagicMock() + twitter.tweet_diff = MagicMock() -def test_twitter_do_nothing_if_entry_has_no_diff(): - # Mock that returns no diff - tweet_diff = MagicMock(name="tweet_diff") - version = MagicMock() - type(version).diff = PropertyMock(return_value=None) + version = MagicMock() + type(version).diff = PropertyMock(return_value=None) - entry = MagicMock() - type(entry).stale = PropertyMock(return_value=True) - entry.get_latest = MagicMock(return_value=version) + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=version) - result = process_entry(entry, None) - assert result["checked"] == 1 - assert result["new"] == 1 - assert entry.get_latest.called - assert not tweet_diff.called + # Test + result = process_entry(entry, None) + # Assert + assert result["checked"] == 1 + assert result["new"] == 1 + entry.get_latest.assert_called_once() + twitter.tweet_diff.assert_not_called() -def test_twitter_do_nothing_if_feed_has_no_token(): - # Mock that returns no diff but it has no token - tweet_diff = MagicMock(name="tweet_diff") - version = MagicMock() - type(version).diff = PropertyMock(return_value=None) + def test_do_not_tweet_if_feed_has_no_token(self): + # Prepare + twitter = MagicMock() + twitter.tweet_diff = MagicMock() - entry = MagicMock() - type(entry).stale = PropertyMock(return_value=True) - entry.get_latest = MagicMock(return_value=version) + version = MagicMock() + type(version).diff = PropertyMock(return_value=None) - result = process_entry(entry, None) - assert result["checked"] == 1 - assert result["new"] == 1 - assert entry.get_latest.called - assert not tweet_diff.called + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=version) + # Test + result = process_entry(entry, None, twitter) -def test_twitter_do_tweet_diff_if_entry_has_diff(): - # Mock that returns a diff - tweet_diff = MagicMock(name="tweet_diff") - version = MagicMock() - type(version).diff = PropertyMock(return_value=MagicMock()) + # Assert + assert result["checked"] == 1 + assert result["new"] == 1 + entry.get_latest.assert_called_once() + twitter.tweet_diff.assert_not_called() - entry = MagicMock() - type(entry).stale = PropertyMock(return_value=True) - entry.get_latest = MagicMock(return_value=version) + def test_do_tweet_if_entry_has_diff(self): + # Prepare + twitter = MagicMock() + twitter.tweet_diff = MagicMock() - token = {"access_token": "test", "access_token_secret": "test"} - result = process_entry(entry, token) + version = MagicMock() + type(version).diff = PropertyMock(return_value=MagicMock()) - assert result["checked"] == 1 - assert result["new"] == 1 - assert entry.get_latest.called - assert tweet_diff.called + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=version) + + # Test + token = {"access_token": "test", "access_token_secret": "test"} + result = process_entry(entry, token, twitter) + + # Assert + assert result["checked"] == 1 + assert result["new"] == 1 + + entry.get_latest.assert_called_once() + twitter.tweet_diff.assert_called_once() From 89b427c51e956ca95da799d2bb265a7a689d7a05 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 09:48:56 -0300 Subject: [PATCH 03/23] Test classes so we can test a group together without running the entire test suite --- test_diffengine.py | 78 ++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/test_diffengine.py b/test_diffengine.py index b240231..aeabc6e 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -154,54 +154,52 @@ def test_fingerprint(): assert _fingerprint("foo’bar") == "foobar" -def test_environment_vars_in_config_file(): - - # test values - public_value = "public value" - private_yaml_key = "${PRIVATE_VAR}" - private_value = "private value" - - # create dot env that that will read - dotenv_file = open(home_path(".env"), "w+") - dotenv_file.write("PRIVATE_VAR=%s\n" % private_value) - dotenv_file.close() - - # create config.yaml that will be read - test_config = { - "example": {"private_value": private_yaml_key, "public_value": public_value} - } - config_file = home_path("config.yaml") - yaml.dump(test_config, open(config_file, "w"), default_flow_style=False) - - # test! - new_config = load_config() - assert new_config["example"]["public_value"] == public_value - assert new_config["example"]["private_value"] != private_yaml_key - assert new_config["example"]["private_value"] == private_value +class EnvVarsTest(TestCase): + def test_config_file_integration(self): + # test values + public_value = "public value" + private_yaml_key = "${PRIVATE_VAR}" + private_value = "private value" + + # create dot env that that will read + dotenv_file = open(home_path(".env"), "w+") + dotenv_file.write("PRIVATE_VAR=%s\n" % private_value) + dotenv_file.close() + # create config.yaml that will be read + test_config = { + "example": {"private_value": private_yaml_key, "public_value": public_value} + } + config_file = home_path("config.yaml") + yaml.dump(test_config, open(config_file, "w"), default_flow_style=False) -def test_geckodriver_when_webdriver_is_not_defined(): - # create config.yaml that will be read - browser = setup_browser() - assert isinstance(browser, webdriver.Firefox) == True + # test! + new_config = load_config() + assert new_config["example"]["public_value"] == public_value + assert new_config["example"]["private_value"] != private_yaml_key + assert new_config["example"]["private_value"] == private_value -def test_raises_when_unknown_webdriver(): - with pytest.raises(UnknownWebdriverError): +class WebdriverTest(TestCase): + def test_geckodriver_when_webdriver_is_not_defined(self): # create config.yaml that will be read - setup_browser("wrong_engine") - + browser = setup_browser() + assert isinstance(browser, webdriver.Firefox) == True -def test_webdriver_is_geckodriver(): - # create config.yaml that will be read - browser = setup_browser("geckodriver") - assert isinstance(browser, webdriver.Firefox) == True + def test_raises_when_unknown_webdriver(self): + with pytest.raises(UnknownWebdriverError): + # create config.yaml that will be read + setup_browser("wrong_engine") + def test_webdriver_is_geckodriver(self): + # create config.yaml that will be read + browser = setup_browser("geckodriver") + assert isinstance(browser, webdriver.Firefox) == True -def test_webdriver_is_chromedriver(): - # create config.yaml that will be read - browser = setup_browser("chromedriver") - assert isinstance(browser, webdriver.Chrome) == True + def test_webdriver_is_chromedriver(self): + # create config.yaml that will be read + browser = setup_browser("chromedriver") + assert isinstance(browser, webdriver.Chrome) == True class EntryTest(TestCase): From ee5d316ce2b5448fc78da6af1d4d7073e0aa8e41 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 09:54:02 -0300 Subject: [PATCH 04/23] Test not new if entry retrievement fails --- test_diffengine.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/test_diffengine.py b/test_diffengine.py index aeabc6e..d97af26 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -209,11 +209,25 @@ def test_stale_is_skipped(self): type(entry).stale = PropertyMock(return_value=False) # Test - result = process_entry(entry) + result = process_entry(entry, None, None) # Assert assert result["skipped"] == 1 + def test_raise_if_entry_retrieve_fails(self): + # Prepare + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(side_effect=Exception) + + # Test + result = process_entry(entry, None, None) + + # Assert + entry.get_latest.assert_called_once() + assert result["checked"] == 1 + assert result["new"] == 0 + def test_do_not_tweet_if_entry_has_no_diff(self): # Prepare twitter = MagicMock() @@ -227,12 +241,12 @@ def test_do_not_tweet_if_entry_has_no_diff(self): entry.get_latest = MagicMock(return_value=version) # Test - result = process_entry(entry, None) + result = process_entry(entry, None, twitter) # Assert + entry.get_latest.assert_called_once() assert result["checked"] == 1 assert result["new"] == 1 - entry.get_latest.assert_called_once() twitter.tweet_diff.assert_not_called() def test_do_not_tweet_if_feed_has_no_token(self): @@ -251,9 +265,9 @@ def test_do_not_tweet_if_feed_has_no_token(self): result = process_entry(entry, None, twitter) # Assert + entry.get_latest.assert_called_once() assert result["checked"] == 1 assert result["new"] == 1 - entry.get_latest.assert_called_once() twitter.tweet_diff.assert_not_called() def test_do_tweet_if_entry_has_diff(self): @@ -273,8 +287,7 @@ def test_do_tweet_if_entry_has_diff(self): result = process_entry(entry, token, twitter) # Assert + entry.get_latest.assert_called_once() assert result["checked"] == 1 assert result["new"] == 1 - - entry.get_latest.assert_called_once() twitter.tweet_diff.assert_called_once() From 3630fbc1721774b6178294d84e66562896a3ed35 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 09:58:59 -0300 Subject: [PATCH 05/23] Twitter init --- diffengine/twitter.py | 7 ++++--- test_diffengine.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/diffengine/twitter.py b/diffengine/twitter.py index dcfb34e..f256456 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -28,9 +28,10 @@ def __init__(self, config): logging.debug("consumer key/secret not set up for feed") return - self.consumer_key = twitter_config["access_token"] - self.consumer_secret = twitter_config["access_secret"] - auth = tweepy.OAuthHandler(t["consumer_key"], t["consumer_secret"]) + self.consumer_key = twitter_config["consumer_key"] + self.consumer_secret = twitter_config["consumer_secret"] + + auth = tweepy.OAuthHandler(self.consumer_key, self.consumer_key) auth.secure = True self.auth = auth diff --git a/test_diffengine.py b/test_diffengine.py index d97af26..815d745 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -24,6 +24,7 @@ UnknownWebdriverError, process_entry, UA, + Twitter, ) if os.path.isdir("test"): @@ -291,3 +292,8 @@ def test_do_tweet_if_entry_has_diff(self): assert result["checked"] == 1 assert result["new"] == 1 twitter.tweet_diff.assert_called_once() + + +class TweetTest(TestCase): + def test_do_nothing_if_has_no_token(self): + twitter = Twitter() From 32ea592392e4c84c808e69f4e2931a4627723695 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 14:29:03 -0300 Subject: [PATCH 06/23] Twitter handler instantiation --- diffengine/__init__.py | 15 ++++++++++----- diffengine/exceptions.py | 7 +++++++ diffengine/twitter.py | 21 ++++++++------------- test_diffengine.py | 18 +++++++++++++++--- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 27bcf11..5fe5124 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -35,8 +35,8 @@ from urllib.parse import urlparse, urlunparse, parse_qs, urlencode from envyaml import EnvYAML -from diffengine.exceptions import UnknownWebdriverError -from diffengine.twitter import Twitter +from diffengine.exceptions import UnknownWebdriverError, TwitterConfigError +from diffengine.twitter import TwitterHandler home = None config = {} @@ -525,12 +525,17 @@ def main(): # get latest feed entries feed.get_latest() - - twitter = Twitter(config) + try: + twitter_handler = TwitterHandler( + config["consumer_key"], config["consumer_secret"] + ) + except TwitterConfigError as e: + twitter_handler = None + logging.warning("error with Twitter handler. Reason", str(e)) # get latest content for each entry for entry in feed.entries: - result = process_entry(entry, f["twitter"], twitter) + result = process_entry(entry, f["twitter"], twitter_handler) skipped += result["skipped"] checked += result["checked"] new += result["new"] diff --git a/diffengine/exceptions.py b/diffengine/exceptions.py index 6c81a28..f325680 100644 --- a/diffengine/exceptions.py +++ b/diffengine/exceptions.py @@ -10,3 +10,10 @@ def __init__(self, driver): 'webdriver "%s" is not a valid engine. Please indicate one of "chromedriver" or "geckodriver" and restart the process.' % driver ) + + +class TwitterConfigError(RuntimeError): + """Exception raised if the Twitter instance has not the required key and secret""" + + def __init__(self): + self.message = "consumer key/secret not set up for feed." diff --git a/diffengine/twitter.py b/diffengine/twitter.py index f256456..42bfdc1 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -3,6 +3,8 @@ from datetime import datetime +from diffengine.exceptions import TwitterConfigError + def build_text(diff): text = diff.new.title @@ -13,23 +15,16 @@ def build_text(diff): return text -class Twitter: +class TwitterHandler: consumer_key = None consumer_secret = None - def __init__(self, config): - twitter_config = config.get("twitter", None) - if twitter_config is None: - logging.debug("twitter not configured") - return - elif ( - not twitter_config["consumer_key"] or not twitter_config["consumer_secret"] - ): - logging.debug("consumer key/secret not set up for feed") - return + def __init__(self, consumer_key, consumer_secret): + if not consumer_key or not consumer_secret: + raise TwitterConfigError() - self.consumer_key = twitter_config["consumer_key"] - self.consumer_secret = twitter_config["consumer_secret"] + self.consumer_key = consumer_key + self.consumer_secret = consumer_secret auth = tweepy.OAuthHandler(self.consumer_key, self.consumer_key) auth.secure = True diff --git a/test_diffengine.py b/test_diffengine.py index 815d745..8bc89b4 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -24,8 +24,9 @@ UnknownWebdriverError, process_entry, UA, - Twitter, + TwitterHandler, ) +from diffengine.exceptions import TwitterConfigError if os.path.isdir("test"): shutil.rmtree("test") @@ -295,5 +296,16 @@ def test_do_tweet_if_entry_has_diff(self): class TweetTest(TestCase): - def test_do_nothing_if_has_no_token(self): - twitter = Twitter() + def test_raises_if_no_config_set(self): + self.assertRaises(TwitterConfigError, TwitterHandler, None, None) + self.assertRaises(TwitterConfigError, TwitterHandler, "myConsumerKey", None) + self.assertRaises(TwitterConfigError, TwitterHandler, None, "myConsumerSecret") + + try: + TwitterHandler("myConsumerKey", "myConsumerSecret") + except TwitterConfigError: + self.fail("Twitter.__init__ raised TwitterConfigError unexpectedly!") + + def test_do_nothing_if(self): + config = {"twitter": {"consumer_key": "test", "consumer_secret": "test"}} + twitter = TwitterHandler(config) From 40e7f9793d797cbbeca9f4c1baabcbca083cc0e5 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 21:23:00 -0300 Subject: [PATCH 07/23] Twitter handler tests --- diffengine/__init__.py | 8 ++- diffengine/twitter.py | 39 ++++++----- exceptions/twitter.py | 26 +++++++ .../exceptions.py => exceptions/webdriver.py | 7 -- test_diffengine.py | 69 ++++++++++++++++--- 5 files changed, 110 insertions(+), 39 deletions(-) create mode 100644 exceptions/twitter.py rename diffengine/exceptions.py => exceptions/webdriver.py (65%) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 5fe5124..e12520b 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -35,7 +35,8 @@ from urllib.parse import urlparse, urlunparse, parse_qs, urlencode from envyaml import EnvYAML -from diffengine.exceptions import UnknownWebdriverError, TwitterConfigError +from exceptions.webdriver import UnknownWebdriverError +from exceptions.twitter import ConfigNotFoundError, TwitterError from diffengine.twitter import TwitterHandler home = None @@ -529,7 +530,7 @@ def main(): twitter_handler = TwitterHandler( config["consumer_key"], config["consumer_secret"] ) - except TwitterConfigError as e: + except ConfigNotFoundError as e: twitter_handler = None logging.warning("error with Twitter handler. Reason", str(e)) @@ -563,9 +564,10 @@ def process_entry(entry, token=None, twitter=None): result["new"] = 1 if version.diff and token is not None: twitter.tweet_diff(version.diff, token) + except TwitterError as e: + logging.warning("error occurred while trying to tweet", e) except Exception as e: logging.error("unable to get latest", e) - return result return result diff --git a/diffengine/twitter.py b/diffengine/twitter.py index 42bfdc1..24d252d 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -3,16 +3,12 @@ from datetime import datetime -from diffengine.exceptions import TwitterConfigError - - -def build_text(diff): - text = diff.new.title - if len(text) >= 225: - text = text[0:225] + "…" - text += " " + diff.url - - return text +from exceptions.twitter import ( + AlreadyTweetedError, + ConfigNotFoundError, + TokenNotFoundError, + AchiveUrlNotFoundError, +) class TwitterHandler: @@ -21,7 +17,7 @@ class TwitterHandler: def __init__(self, consumer_key, consumer_secret): if not consumer_key or not consumer_secret: - raise TwitterConfigError() + raise ConfigNotFoundError() self.consumer_key = consumer_key self.consumer_secret = consumer_secret @@ -34,6 +30,14 @@ def api(self, token): self.auth.set_access_token(token["access_token"], token["access_token_secret"]) return tweepy.API(self.auth) + def build_text(diff): + text = diff.new.title + if len(text) >= 225: + text = text[0:225] + "…" + text += " " + diff.url + + return text + def tweet_thread(self, entry, first_version, token): if not token: logging.debug("access token/secret not set up for feed") @@ -52,19 +56,16 @@ def tweet_thread(self, entry, first_version, token): first_version.save() return status.id_str - def tweet_diff(self, diff, token): + def tweet_diff(self, diff, token=None): if not token: - logging.debug("access token/secret not set up for feed") - return + raise TokenNotFoundError() elif diff.tweeted: - logging.warning("diff %s has already been tweeted", diff.id) - return + raise AlreadyTweetedError(diff.id) elif not (diff.old.archive_url and diff.new.archive_url): - logging.warning("not tweeting without archive urls") - return + raise AchiveUrlNotFoundError() twitter = self.api(token) - text = build_text(diff) + text = self.build_text(diff) # Check if the thread exists thread_status_id_str = None diff --git a/exceptions/twitter.py b/exceptions/twitter.py new file mode 100644 index 0000000..d5fe6c6 --- /dev/null +++ b/exceptions/twitter.py @@ -0,0 +1,26 @@ +class TwitterError(RuntimeError): + pass + + +class ConfigNotFoundError(TwitterError): + """Exception raised if the Twitter instance has not the required key and secret""" + + def __init__(self): + self.message = "consumer key/secret not set up for feed." + + +class TokenNotFoundError(TwitterError): + """Exception raised if no token is preset""" + + def __init__(self): + self.message = "access token/secret not set up for feed" + + +class AlreadyTweetedError(TwitterError): + def __init__(self, diff_id): + self.message = "diff %s has already been tweeted" % diff_id + + +class AchiveUrlNotFoundError(TwitterError): + def __init__(self): + self.message = "not tweeting without archive urls" diff --git a/diffengine/exceptions.py b/exceptions/webdriver.py similarity index 65% rename from diffengine/exceptions.py rename to exceptions/webdriver.py index f325680..6c81a28 100644 --- a/diffengine/exceptions.py +++ b/exceptions/webdriver.py @@ -10,10 +10,3 @@ def __init__(self, driver): 'webdriver "%s" is not a valid engine. Please indicate one of "chromedriver" or "geckodriver" and restart the process.' % driver ) - - -class TwitterConfigError(RuntimeError): - """Exception raised if the Twitter instance has not the required key and secret""" - - def __init__(self): - self.message = "consumer key/secret not set up for feed." diff --git a/test_diffengine.py b/test_diffengine.py index 8bc89b4..3d61f26 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -26,7 +26,12 @@ UA, TwitterHandler, ) -from diffengine.exceptions import TwitterConfigError +from exceptions.twitter import ( + ConfigNotFoundError, + TokenNotFoundError, + AlreadyTweetedError, + AchiveUrlNotFoundError, +) if os.path.isdir("test"): shutil.rmtree("test") @@ -295,17 +300,61 @@ def test_do_tweet_if_entry_has_diff(self): twitter.tweet_diff.assert_called_once() -class TweetTest(TestCase): +class TwitterHandlerTest(TestCase): def test_raises_if_no_config_set(self): - self.assertRaises(TwitterConfigError, TwitterHandler, None, None) - self.assertRaises(TwitterConfigError, TwitterHandler, "myConsumerKey", None) - self.assertRaises(TwitterConfigError, TwitterHandler, None, "myConsumerSecret") + self.assertRaises(ConfigNotFoundError, TwitterHandler, None, None) + self.assertRaises(ConfigNotFoundError, TwitterHandler, "myConsumerKey", None) + self.assertRaises(ConfigNotFoundError, TwitterHandler, None, "myConsumerSecret") try: TwitterHandler("myConsumerKey", "myConsumerSecret") - except TwitterConfigError: - self.fail("Twitter.__init__ raised TwitterConfigError unexpectedly!") + except ConfigNotFoundError: + self.fail("Twitter.__init__ raised ConfigNotFoundError unexpectedly!") + + def test_raises_if_no_token_provided(self): + diff = MagicMock() + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + self.assertRaises(TokenNotFoundError, twitter.tweet_diff, diff, None) + + def test_raises_if_already_tweeted(self): + diff = MagicMock() + type(diff).tweeted = PropertyMock(return_value=True) + + token = { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + } + + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + self.assertRaises(AlreadyTweetedError, twitter.tweet_diff, diff, token) + + def test_raises_if_not_all_archive_urls_are_present(self): + + old = MagicMock() + type(old).archive_url = None - def test_do_nothing_if(self): - config = {"twitter": {"consumer_key": "test", "consumer_secret": "test"}} - twitter = TwitterHandler(config) + new = MagicMock() + type(new).archive_url = None + + diff = MagicMock() + type(diff).tweeted = PropertyMock(return_value=False) + type(diff).old = old + type(diff).new = new + + token = { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + } + + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + self.assertRaises(AchiveUrlNotFoundError, twitter.tweet_diff, diff, token) + + type(old).archive_url = PropertyMock(return_value="http://test.url/old") + self.assertRaises(AchiveUrlNotFoundError, twitter.tweet_diff, diff, token) + + type(new).archive_url = PropertyMock(return_value="http://test.url/new") + + try: + twitter.tweet_diff(diff, token) + except AchiveUrlNotFoundError: + self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!") From 0cda96a4d8a76aae5c1b7d708cb91e7501e85c45 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 22:11:12 -0300 Subject: [PATCH 08/23] Assert create/update thread --- diffengine/twitter.py | 3 +- test_diffengine.py | 79 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/diffengine/twitter.py b/diffengine/twitter.py index 24d252d..8194095 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -30,12 +30,11 @@ def api(self, token): self.auth.set_access_token(token["access_token"], token["access_token_secret"]) return tweepy.API(self.auth) - def build_text(diff): + def build_text(self, diff): text = diff.new.title if len(text) >= 225: text = text[0:225] + "…" text += " " + diff.url - return text def tweet_thread(self, entry, first_version, token): diff --git a/test_diffengine.py b/test_diffengine.py index 3d61f26..df58ea7 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -9,7 +9,7 @@ import shutil from unittest import TestCase -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from unittest.mock import PropertyMock from diffengine import ( @@ -329,18 +329,7 @@ def test_raises_if_already_tweeted(self): self.assertRaises(AlreadyTweetedError, twitter.tweet_diff, diff, token) def test_raises_if_not_all_archive_urls_are_present(self): - - old = MagicMock() - type(old).archive_url = None - - new = MagicMock() - type(new).archive_url = None - - diff = MagicMock() - type(diff).tweeted = PropertyMock(return_value=False) - type(diff).old = old - type(diff).new = new - + diff = get_mocked_diff(False) token = { "access_token": "myAccessToken", "access_token_secret": "myAccessTokenSecret", @@ -349,12 +338,70 @@ def test_raises_if_not_all_archive_urls_are_present(self): twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") self.assertRaises(AchiveUrlNotFoundError, twitter.tweet_diff, diff, token) - type(old).archive_url = PropertyMock(return_value="http://test.url/old") + type(diff.old).archive_url = PropertyMock(return_value="http://test.url/old") self.assertRaises(AchiveUrlNotFoundError, twitter.tweet_diff, diff, token) - type(new).archive_url = PropertyMock(return_value="http://test.url/new") - + type(diff.new).archive_url = PropertyMock(return_value="http://test.url/new") try: twitter.tweet_diff(diff, token) except AchiveUrlNotFoundError: self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!") + + @patch("diffengine.TwitterHandler.tweet_thread") + def test_create_thread_if_old_entry_has_no_related_tweet(self, mocked_tweet_thread): + + entry = MagicMock() + type(entry).tweet_status_id_str = PropertyMock(return_value=None) + + diff = get_mocked_diff() + type(diff.old).entry = entry + + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + twitter.tweet_diff( + diff, + { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + }, + ) + + mocked_tweet_thread.assert_called_once() + + @patch("diffengine.TwitterHandler.tweet_thread") + def test_update_thread_if_old_entry_has_related_tweet(self, mocked_tweet_thread): + + entry = MagicMock() + type(entry).tweet_status_id_str = PropertyMock(return_value="1234567890") + + diff = get_mocked_diff() + type(diff.old).entry = entry + + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + twitter.tweet_diff( + diff, + { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + }, + ) + + mocked_tweet_thread.assert_not_called() + + +def get_mocked_diff(with_archive_urls=True): + old = MagicMock() + type(old).archive_url = None + + new = MagicMock() + type(new).archive_url = None + + diff = MagicMock() + type(diff).tweeted = PropertyMock(return_value=False) + type(diff).old = old + type(diff).new = new + + if with_archive_urls: + type(old).archive_url = PropertyMock(return_value="http://test.url/old") + type(new).archive_url = PropertyMock(return_value="http://test.url/new") + + return diff From 03948d76e039ddc0376b162b0748d0f19c3cec9b Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 22:54:50 -0300 Subject: [PATCH 09/23] TwitterHandler.tweet_diff tests finished --- test_diffengine.py | 62 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/test_diffengine.py b/test_diffengine.py index df58ea7..275eecb 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -347,7 +347,6 @@ def test_raises_if_not_all_archive_urls_are_present(self): except AchiveUrlNotFoundError: self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!") - @patch("diffengine.TwitterHandler.tweet_thread") def test_create_thread_if_old_entry_has_no_related_tweet(self, mocked_tweet_thread): entry = MagicMock() @@ -387,6 +386,67 @@ def test_update_thread_if_old_entry_has_related_tweet(self, mocked_tweet_thread) mocked_tweet_thread.assert_not_called() + class MockedStatus(MagicMock): + id_str = PropertyMock(return_value="1234567890") + + @patch("tweepy.OAuthHandler.get_username", return_value="test_user") + @patch("tweepy.API.update_with_media", return_value=MockedStatus) + def test_update_thread_if_old_entry_has_related_tweet( + self, mocked_update_with_media, mocked_get_username + ): + entry = MagicMock() + type(entry).tweet_status_id_str = PropertyMock(return_value="1234567890") + + diff = get_mocked_diff() + type(diff.old).entry = entry + type(diff.new).tweet_status_id_str = PropertyMock() + type(diff.new).save = MagicMock() + type(diff).tweeted = PropertyMock(return_value=False) + type(diff).save = MagicMock() + + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + twitter.tweet_diff( + diff, + { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + }, + ) + + mocked_update_with_media.assert_called_once() + mocked_get_username.assert_called_once() + diff.new.save.assert_called_once() + diff.save.assert_called_once() + + @patch("tweepy.OAuthHandler.get_username", return_value="test_user") + @patch("tweepy.API.update_with_media", side_effect=Exception) + def test_raise_when_thread_tweet_fails( + self, mocked_update_with_media, mocked_get_username + ): + entry = MagicMock() + type(entry).tweet_status_id_str = PropertyMock(return_value="1234567890") + + diff = get_mocked_diff() + type(diff.old).entry = entry + type(diff.new).tweet_status_id_str = PropertyMock() + type(diff.new).save = MagicMock() + type(diff).tweeted = PropertyMock(return_value=False) + type(diff).save = MagicMock() + + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + twitter.tweet_diff( + diff, + { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + }, + ) + + mocked_update_with_media.assert_called_once() + mocked_get_username.assert_not_called() + diff.new.save.assert_not_called() + diff.save.assert_not_called() + def get_mocked_diff(with_archive_urls=True): old = MagicMock() From 3e6c063be9285378370d3a71bc0ef67261c28f53 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 23:07:00 -0300 Subject: [PATCH 10/23] TwitterHandler.tweet_diff tests finished --- test_diffengine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_diffengine.py b/test_diffengine.py index 275eecb..c2bcf55 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -347,6 +347,7 @@ def test_raises_if_not_all_archive_urls_are_present(self): except AchiveUrlNotFoundError: self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!") + @patch("diffengine.TwitterHandler.tweet_thread") def test_create_thread_if_old_entry_has_no_related_tweet(self, mocked_tweet_thread): entry = MagicMock() From adc02b7621394befec81407e68cec86b67fd79c0 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Wed, 29 Apr 2020 23:17:25 -0300 Subject: [PATCH 11/23] TwitterHandler.create_thread failure test --- diffengine/twitter.py | 39 +++++++++++++++++---------------------- exceptions/twitter.py | 8 ++++++++ test_diffengine.py | 18 ++++++++++++++++++ 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/diffengine/twitter.py b/diffengine/twitter.py index 8194095..ad43663 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -8,6 +8,7 @@ ConfigNotFoundError, TokenNotFoundError, AchiveUrlNotFoundError, + UpdateStatusError, ) @@ -37,23 +38,19 @@ def build_text(self, diff): text += " " + diff.url return text - def tweet_thread(self, entry, first_version, token): - if not token: - logging.debug("access token/secret not set up for feed") - return - elif entry.tweet_status_id_str: - logging.warning("entry %s has already been tweeted", entry.id) - return - - twitter = self.api(token) - status = twitter.update_status(entry.url) - entry.tweet_status_id_str = status.id_str - entry.save() - - # Save the entry status_id inside the first entryVersion - first_version.tweet_status_id_str = status.id_str - first_version.save() - return status.id_str + def create_thread(self, entry, first_version, token): + try: + twitter = self.api(token) + status = twitter.update_status(entry.url) + entry.tweet_status_id_str = status.id_str + entry.save() + + # Save the entry status_id inside the first entryVersion + first_version.tweet_status_id_str = status.id_str + first_version.save() + return status.id_str + except Exception as e: + raise UpdateStatusError(entry) def tweet_diff(self, diff, token=None): if not token: @@ -70,17 +67,15 @@ def tweet_diff(self, diff, token=None): thread_status_id_str = None if diff.old.entry.tweet_status_id_str is None: try: - thread_status_id_str = self.tweet_thread( + thread_status_id_str = self.create_thread( diff.old.entry, diff.old, token ) logging.info( "created thread https://twitter/%s/status/%s" % (self.auth.get_username(), thread_status_id_str) ) - except Exception as e: - logging.error( - "could not create thread on entry %s" % diff.old.entry.url, e - ) + except UpdateStatusError as e: + logging.error(str(e)) else: thread_status_id_str = diff.old.tweet_status_id_str diff --git a/exceptions/twitter.py b/exceptions/twitter.py index d5fe6c6..41c4ef8 100644 --- a/exceptions/twitter.py +++ b/exceptions/twitter.py @@ -24,3 +24,11 @@ def __init__(self, diff_id): class AchiveUrlNotFoundError(TwitterError): def __init__(self): self.message = "not tweeting without archive urls" + + +class UpdateStatusError(TwitterError): + def __init__(self, entry): + self.message = "could not create thread on entry id %s, url %s" % ( + entry.id, + entry.url, + ) diff --git a/test_diffengine.py b/test_diffengine.py index c2bcf55..ce6aa8d 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -31,6 +31,7 @@ TokenNotFoundError, AlreadyTweetedError, AchiveUrlNotFoundError, + UpdateStatusError, ) if os.path.isdir("test"): @@ -448,6 +449,23 @@ def test_raise_when_thread_tweet_fails( diff.new.save.assert_not_called() diff.save.assert_not_called() + @patch("tweepy.API.update_status", side_effect=Exception) + def test_create_thread_failure(self, mocked_update_status): + entry = MagicMock() + version = MagicMock() + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + self.assertRaises( + UpdateStatusError, + twitter.create_thread, + entry, + version, + { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + }, + ) + mocked_update_status.assert_called_once() + def get_mocked_diff(with_archive_urls=True): old = MagicMock() From fc323c6a7d8ce70edd7875c0f2dece7a502936c0 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 00:47:36 -0300 Subject: [PATCH 12/23] TwitterHandler.create_thread success test --- test_diffengine.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_diffengine.py b/test_diffengine.py index ce6aa8d..fd4a791 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -466,6 +466,29 @@ def test_create_thread_failure(self, mocked_update_status): ) mocked_update_status.assert_called_once() + @patch("tweepy.API.update_status", return_value=MockedStatus) + def test_create_thread_success(self, mocked_update_status): + entry = MagicMock() + type(entry).save = MagicMock() + version = MagicMock() + type(version).save = MagicMock() + twitter = TwitterHandler("myConsumerKey", "myConsumerSecret") + + status_id_str = twitter.create_thread( + entry, + version, + { + "access_token": "myAccessToken", + "access_token_secret": "myAccessTokenSecret", + }, + ) + + self.assertEqual(status_id_str, mocked_update_status.return_value.id_str) + + mocked_update_status.assert_called_once() + entry.save.assert_called_once() + version.save.assert_called_once() + def get_mocked_diff(with_archive_urls=True): old = MagicMock() From 07b3404f179a9fd109e4e6b1f9c8abf034dfa4a8 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 01:47:27 -0300 Subject: [PATCH 13/23] Test adaptations --- test_diffengine.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test_diffengine.py b/test_diffengine.py index fd4a791..9209522 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -348,8 +348,11 @@ def test_raises_if_not_all_archive_urls_are_present(self): except AchiveUrlNotFoundError: self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!") - @patch("diffengine.TwitterHandler.tweet_thread") - def test_create_thread_if_old_entry_has_no_related_tweet(self, mocked_tweet_thread): + @patch("tweepy.OAuthHandler.get_username", return_value="test_user") + @patch("diffengine.TwitterHandler.create_thread") + def test_create_thread_if_old_entry_has_no_related_tweet( + self, mocked_create_thread, mocked_get_username + ): entry = MagicMock() type(entry).tweet_status_id_str = PropertyMock(return_value=None) @@ -366,10 +369,14 @@ def test_create_thread_if_old_entry_has_no_related_tweet(self, mocked_tweet_thre }, ) - mocked_tweet_thread.assert_called_once() + mocked_create_thread.assert_called_once() + mocked_get_username.assert_called_once() - @patch("diffengine.TwitterHandler.tweet_thread") - def test_update_thread_if_old_entry_has_related_tweet(self, mocked_tweet_thread): + @patch("tweepy.OAuthHandler.get_username", return_value="test_user") + @patch("diffengine.TwitterHandler.create_thread") + def test_update_thread_if_old_entry_has_related_tweet( + self, mocked_create_thread, mocked_get_username + ): entry = MagicMock() type(entry).tweet_status_id_str = PropertyMock(return_value="1234567890") @@ -386,7 +393,8 @@ def test_update_thread_if_old_entry_has_related_tweet(self, mocked_tweet_thread) }, ) - mocked_tweet_thread.assert_not_called() + mocked_create_thread.assert_not_called() + mocked_get_username.assert_called_once() class MockedStatus(MagicMock): id_str = PropertyMock(return_value="1234567890") From bf20cddc3607327723ad5b1cbd942ef5338ffa78 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 08:54:51 -0300 Subject: [PATCH 14/23] travis ci github integration From 8525f35fd245cc1be7c1cf891e4779bf0f7909fe Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 09:15:46 -0300 Subject: [PATCH 15/23] Config KeyError exception catch --- diffengine/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index e12520b..9b0c126 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -517,6 +517,18 @@ def main(): start_time = datetime.utcnow() logging.info("starting up with home=%s", home) + try: + twitter_config = config.get("twitter") + twitter_handler = TwitterHandler( + twitter_config["consumer_key"], twitter_config["consumer_secret"] + ) + except ConfigNotFoundError as e: + twitter_handler = None + logging.warning("error when creating Twitter Handler. Reason", str(e)) + except KeyError as e: + twitter_handler = None + logging.warning("the twitter keys are not present in config. Reason", str(e)) + checked = skipped = new = 0 for f in config.get("feeds", []): @@ -526,13 +538,6 @@ def main(): # get latest feed entries feed.get_latest() - try: - twitter_handler = TwitterHandler( - config["consumer_key"], config["consumer_secret"] - ) - except ConfigNotFoundError as e: - twitter_handler = None - logging.warning("error with Twitter handler. Reason", str(e)) # get latest content for each entry for entry in feed.entries: From 9b16292d8589f36389d26e24d04558f0f82aa4b3 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 09:31:01 -0300 Subject: [PATCH 16/23] process_entry case where version is None --- diffengine/__init__.py | 7 ++++--- test_diffengine.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 9b0c126..03032b3 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -566,9 +566,10 @@ def process_entry(entry, token=None, twitter=None): result["checked"] = 1 try: version = entry.get_latest() - result["new"] = 1 - if version.diff and token is not None: - twitter.tweet_diff(version.diff, token) + if version: + result["new"] = 1 + if version.diff and token is not None: + twitter.tweet_diff(version.diff, token) except TwitterError as e: logging.warning("error occurred while trying to tweet", e) except Exception as e: diff --git a/test_diffengine.py b/test_diffengine.py index 9209522..976207d 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -236,6 +236,24 @@ def test_raise_if_entry_retrieve_fails(self): assert result["checked"] == 1 assert result["new"] == 0 + def test_get_none_if_no_new_version(self): + # Prepare + twitter = MagicMock() + twitter.tweet_diff = MagicMock() + + entry = MagicMock() + type(entry).stale = PropertyMock(return_value=True) + entry.get_latest = MagicMock(return_value=None) + + # Test + result = process_entry(entry, None, twitter) + + # Assert + entry.get_latest.assert_called_once() + assert result["checked"] == 1 + assert result["new"] == 0 + twitter.tweet_diff.assert_not_called() + def test_do_not_tweet_if_entry_has_no_diff(self): # Prepare twitter = MagicMock() From 6effef85d283b49f3d578360783e7f668fd87100 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 12:22:03 -0300 Subject: [PATCH 17/23] Catch specific tweeting errors and add tweet_status_id_str field to Entry and EntryVersion --- diffengine/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 03032b3..1798299 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -99,6 +99,7 @@ class Entry(BaseModel): url = CharField() created = DateTimeField(default=datetime.utcnow) checked = DateTimeField(default=datetime.utcnow) + tweet_status_id_str = CharField() @property def feeds(self): @@ -224,6 +225,7 @@ class EntryVersion(BaseModel): created = DateTimeField(default=datetime.utcnow) archive_url = CharField(null=True) entry = ForeignKeyField(Entry, backref="versions") + tweet_status_id_str = CharField() @property def diff(self): @@ -569,9 +571,12 @@ def process_entry(entry, token=None, twitter=None): if version: result["new"] = 1 if version.diff and token is not None: - twitter.tweet_diff(version.diff, token) - except TwitterError as e: - logging.warning("error occurred while trying to tweet", e) + try: + twitter.tweet_diff(version.diff, token) + except TwitterError as e: + logging.warning("error occurred while trying to tweet", e) + except Exception as e: + logging.error("unknown error when tweeting diff", e) except Exception as e: logging.error("unable to get latest", e) return result From ede2eb41c8af6840d7178deb64613a9507cf1fcb Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 12:23:49 -0300 Subject: [PATCH 18/23] The field tweet_status_id_str is nullable --- diffengine/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 1798299..3f5226e 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -99,7 +99,7 @@ class Entry(BaseModel): url = CharField() created = DateTimeField(default=datetime.utcnow) checked = DateTimeField(default=datetime.utcnow) - tweet_status_id_str = CharField() + tweet_status_id_str = CharField(null=True) @property def feeds(self): @@ -225,7 +225,7 @@ class EntryVersion(BaseModel): created = DateTimeField(default=datetime.utcnow) archive_url = CharField(null=True) entry = ForeignKeyField(Entry, backref="versions") - tweet_status_id_str = CharField() + tweet_status_id_str = CharField(null=True) @property def diff(self): From 43b24676521750de99705c430a99a5c92aa72525 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 14:03:17 -0300 Subject: [PATCH 19/23] Pass the consumer secret --- diffengine/twitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffengine/twitter.py b/diffengine/twitter.py index ad43663..2c8c5ef 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -23,7 +23,7 @@ def __init__(self, consumer_key, consumer_secret): self.consumer_key = consumer_key self.consumer_secret = consumer_secret - auth = tweepy.OAuthHandler(self.consumer_key, self.consumer_key) + auth = tweepy.OAuthHandler(self.consumer_key, self.consumer_secret) auth.secure = True self.auth = auth From 322cc0a629236128d2924e12a779e7be12a11d2c Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 14:11:09 -0300 Subject: [PATCH 20/23] Thread creation url fix. Now when no diff is detected, indicates the version id. Useful for debugging purposes --- diffengine/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index 3f5226e..a7fd403 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -198,7 +198,12 @@ def get_latest(self): logging.debug("found new version %s", old.entry.url) diff = Diff.create(old=old, new=new) if not diff.generate(): - logging.warn("html diff showed no changes: %s", self.url) + logging.warn( + "html diff showed no changes between versions #%s and #%s: %s", + old.id, + new.id, + self.url, + ) new.delete() new = None else: From a877076460aac261e2d4c3cb731d8e13846ddd27 Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 14:13:46 -0300 Subject: [PATCH 21/23] TODO --- diffengine/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/diffengine/__init__.py b/diffengine/__init__.py index a7fd403..b8a78e8 100755 --- a/diffengine/__init__.py +++ b/diffengine/__init__.py @@ -315,6 +315,7 @@ def snap(url): snap(self.old.archive_url), snap(self.new.archive_url), self.old.url ) + # TODO: configurable option for deleting the diffs after some time (1 week?) def generate(self): if self._generate_diff_html(): self._generate_diff_images() From c38cd98a9a525e1224a235d97aa6d31f7affecef Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 14:48:49 -0300 Subject: [PATCH 22/23] thread link fix --- diffengine/twitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffengine/twitter.py b/diffengine/twitter.py index 2c8c5ef..05e2d88 100644 --- a/diffengine/twitter.py +++ b/diffengine/twitter.py @@ -71,7 +71,7 @@ def tweet_diff(self, diff, token=None): diff.old.entry, diff.old, token ) logging.info( - "created thread https://twitter/%s/status/%s" + "created thread https://twitter.com/%s/status/%s" % (self.auth.get_username(), thread_status_id_str) ) except UpdateStatusError as e: From 9cb4ad67eddb9d6971853c15e8125747450c6b2b Mon Sep 17 00:00:00 2001 From: Nahuel Sotelo Date: Thu, 30 Apr 2020 20:36:46 -0300 Subject: [PATCH 23/23] Tests fixes --- test_diffengine.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/test_diffengine.py b/test_diffengine.py index 976207d..6231ab7 100644 --- a/test_diffengine.py +++ b/test_diffengine.py @@ -1,3 +1,4 @@ +import logging import os import re @@ -211,6 +212,12 @@ def test_webdriver_is_chromedriver(self): class EntryTest(TestCase): + def setUp(self) -> None: + logging.disable(logging.CRITICAL) + + def tearDown(self) -> None: + logging.disable(logging.NOTSET) + def test_stale_is_skipped(self): # Prepare entry = MagicMock() @@ -226,7 +233,7 @@ def test_raise_if_entry_retrieve_fails(self): # Prepare entry = MagicMock() type(entry).stale = PropertyMock(return_value=True) - entry.get_latest = MagicMock(side_effect=Exception) + entry.get_latest = MagicMock(side_effect=Exception("TEST")) # Test result = process_entry(entry, None, None) @@ -320,6 +327,12 @@ def test_do_tweet_if_entry_has_diff(self): class TwitterHandlerTest(TestCase): + def setUp(self) -> None: + logging.disable(logging.CRITICAL) + + def tearDown(self) -> None: + logging.disable(logging.NOTSET) + def test_raises_if_no_config_set(self): self.assertRaises(ConfigNotFoundError, TwitterHandler, None, None) self.assertRaises(ConfigNotFoundError, TwitterHandler, "myConsumerKey", None) @@ -366,10 +379,14 @@ def test_raises_if_not_all_archive_urls_are_present(self): except AchiveUrlNotFoundError: self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!") + class MockedStatus(MagicMock): + id_str = PropertyMock(return_value="1234567890") + @patch("tweepy.OAuthHandler.get_username", return_value="test_user") + @patch("tweepy.API.update_with_media", return_value=MockedStatus) @patch("diffengine.TwitterHandler.create_thread") def test_create_thread_if_old_entry_has_no_related_tweet( - self, mocked_create_thread, mocked_get_username + self, mocked_create_thread, mocked_update_with_media, mocked_get_username ): entry = MagicMock() @@ -388,7 +405,8 @@ def test_create_thread_if_old_entry_has_no_related_tweet( ) mocked_create_thread.assert_called_once() - mocked_get_username.assert_called_once() + mocked_update_with_media.assert_called_once() + mocked_get_username.assert_called() @patch("tweepy.OAuthHandler.get_username", return_value="test_user") @patch("diffengine.TwitterHandler.create_thread") @@ -414,9 +432,6 @@ def test_update_thread_if_old_entry_has_related_tweet( mocked_create_thread.assert_not_called() mocked_get_username.assert_called_once() - class MockedStatus(MagicMock): - id_str = PropertyMock(return_value="1234567890") - @patch("tweepy.OAuthHandler.get_username", return_value="test_user") @patch("tweepy.API.update_with_media", return_value=MockedStatus) def test_update_thread_if_old_entry_has_related_tweet(