Database is now connected through proxy and inferred by its url defin…

…ed at config `db`

diffengine/__init__.py

@@ -27,29 +27,35 @@
 from diffengine.utils import request_pin_to_user_and_get_token
 
 from exceptions.webdriver import UnknownWebdriverError
-from exceptions.sendgrid import (
-    ConfigNotFoundError as SGConfigNotFoundError,
-    SendgridError,
-)
-from exceptions.twitter import ConfigNotFoundError, TwitterError
+from exceptions.sendgrid import SendgridConfigNotFoundError, SendgridError
+from exceptions.twitter import TwitterConfigNotFoundError, TwitterError
 
-from peewee import *
-from playhouse.migrate import SqliteMigrator, migrate
 from datetime import datetime
+from peewee import (
+    DatabaseProxy,
+    CharField,
+    DateTimeField,
+    OperationalError,
+    ForeignKeyField,
+    Model,
+    SqliteDatabase,
+)
+from playhouse.db_url import connect
+from playhouse.migrate import SqliteMigrator, migrate
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options as ChromeOptions
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
 from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
 
 home = None
 config = {}
-db = SqliteDatabase(None)
+database = DatabaseProxy()
 browser = None
 
 
 class BaseModel(Model):
     class Meta:
-        database = db
+        database = database
 
 
 class Feed(BaseModel):
@@ -403,17 +409,20 @@ def get_auth_link_and_show_token():
 
 
 def setup_db():
-    global home, db
-    db_file = config.get("db", os.path.join(home, "diffengine.db"))
-    logging.debug("connecting to db %s", db_file)
-    db.init(db_file)
-    db.connect()
-    db.create_tables([Feed, Entry, FeedEntry, EntryVersion, Diff], safe=True)
-    try:
-        migrator = SqliteMigrator(db)
-        migrate(migrator.add_index("entryversion", ("url",), False))
-    except OperationalError as e:
-        logging.debug(e)
+    global home, database
+    database_url = config.get("db", "sqlite:///diffengine.db")
+    logging.debug("connecting to db %s", database_url)
+    database_handler = connect(database_url)
+    database.initialize(database_handler)
+    database.connect()
+    database.create_tables([Feed, Entry, FeedEntry, EntryVersion, Diff], safe=True)
+
+    if isinstance(database_handler, SqliteDatabase):
+        try:
+            migrator = SqliteMigrator(database)
+            migrate(migrator.add_index("entryversion", ("url",), False))
+        except OperationalError as e:
+            logging.debug(e)
 
 
 def chromedriver_browser(executable_path, binary_location):
@@ -483,7 +492,7 @@ def main():
         twitter_handler = TwitterHandler(
             twitter_config["consumer_key"], twitter_config["consumer_secret"]
         )
-    except ConfigNotFoundError as e:
+    except TwitterConfigNotFoundError as e:
         twitter_handler = None
         logging.warning("error when creating Twitter Handler. Reason", str(e))
     except KeyError as e:
@@ -547,7 +556,7 @@ def process_entry(entry, feed_config, twitter=None, sendgrid=None, lang={}):
                             version.diff, feed_config.get("sendgrid", {})
                         )
 
-                    except SGConfigNotFoundError as e:
+                    except SendgridConfigNotFoundError as e:
                         logging.error(
                             "Missing configuration values for publishing entry %s",
                             entry.url,

diffengine/database.py

@@ -7,14 +7,14 @@
 DATABASE_URL = os.getenv("DATABASE_URL")
 if DATABASE_URL is not None:
     print("defined database. Using PostgreSQL.")
-    db = connect(DATABASE_URL)
+    database = connect(DATABASE_URL)
 else:
     print("no database defined, using SQLite.")
-    db = SqliteDatabase(None)
+    database = SqliteDatabase(None)
 
 
 def setup_db():
-    global db
+    global database
 
     # If it's local, it needs to be init
     if DATABASE_URL is None:

diffengine/sendgrid.py

@@ -5,8 +5,8 @@
 
 from exceptions.sendgrid import (
     AlreadyEmailedError,
-    ConfigNotFoundError,
-    ArchiveUrlNotFoundError,
+    SendgridConfigNotFoundError,
+    SendgridArchiveUrlNotFoundError,
 )
 
 
@@ -43,13 +43,13 @@ def publish_diff(self, diff, feed_config):
         if diff.emailed:
             raise AlreadyEmailedError(diff.id)
         elif not (diff.old.archive_url and diff.new.archive_url):
-            raise ArchiveUrlNotFoundError()
+            raise SendgridArchiveUrlNotFoundError()
 
         api_token = feed_config.get("api_token", self.api_token)
         sender = feed_config.get("sender", self.sender)
         receivers = feed_config.get("receivers", self.receivers)
         if not all([api_token, sender, receivers]):
-            raise ConfigNotFoundError
+            raise SendgridConfigNotFoundError
 
         subject = self.build_subject(diff)
         message = Mail(

diffengine/twitter.py

@@ -6,9 +6,9 @@
 from diffengine.text_builder import build_text
 from exceptions.twitter import (
     AlreadyTweetedError,
-    ConfigNotFoundError,
+    TwitterConfigNotFoundError,
     TokenNotFoundError,
-    AchiveUrlNotFoundError,
+    TwitterAchiveUrlNotFoundError,
     UpdateStatusError,
 )
 
@@ -19,7 +19,7 @@ class TwitterHandler:
 
     def __init__(self, consumer_key, consumer_secret):
         if not consumer_key or not consumer_secret:
-            raise ConfigNotFoundError()
+            raise TwitterConfigNotFoundError()
 
         self.consumer_key = consumer_key
         self.consumer_secret = consumer_secret
@@ -59,7 +59,7 @@ def tweet_diff(self, diff, token=None, lang={}):
         elif diff.tweeted:
             raise AlreadyTweetedError(diff)
         elif not (diff.old.archive_url and diff.new.archive_url):
-            raise AchiveUrlNotFoundError(diff)
+            raise TwitterAchiveUrlNotFoundError(diff)
 
         twitter = self.api(token)
         text = build_text(diff, lang)

diffengine/utils.py

@@ -1,4 +1,6 @@
+import os
 import tweepy
+import yaml
 
 
 def request_pin_to_user_and_get_token(consumer_key, consumer_secret):
@@ -11,3 +13,12 @@ def request_pin_to_user_and_get_token(consumer_key, consumer_secret):
     input("Visit %s in your browser and hit enter." % auth_url)
     pin = input("What is your PIN: ")
     return auth.get_access_token(verifier=pin)
+
+
+def generate_config(home, content):
+    config_file = os.path.join(home, "config.yaml")
+
+    if not os.path.isdir(home):
+        os.makedirs(home)
+
+    yaml.dump(content, open(config_file, "w"), default_flow_style=False)

exceptions/sendgrid.py

@@ -2,7 +2,7 @@ class SendgridError(RuntimeError):
     pass
 
 
-class ConfigNotFoundError(SendgridError):
+class SendgridConfigNotFoundError(SendgridError):
     """Exception raised if the Sendgrid instance has not the API key"""
 
     def __init__(self):
@@ -14,6 +14,6 @@ def __init__(self, diff_id):
         self.message = "diff %s was already emailed with sendgrid " % diff_id
 
 
-class ArchiveUrlNotFoundError(SendgridError):
+class SendgridArchiveUrlNotFoundError(SendgridError):
     def __init__(self):
         self.message = "not publishing without archive urls"

exceptions/twitter.py

@@ -2,7 +2,7 @@ class TwitterError(RuntimeError):
     pass
 
 
-class ConfigNotFoundError(TwitterError):
+class TwitterConfigNotFoundError(TwitterError):
     """Exception raised if the Twitter instance has not the required key and secret"""
 
     def __init__(self):
@@ -21,7 +21,7 @@ def __init__(self, diff):
         self.message = "diff %s has already been tweeted" % diff.id
 
 
-class AchiveUrlNotFoundError(TwitterError):
+class TwitterAchiveUrlNotFoundError(TwitterError):
     def __init__(self, diff):
         self.message = "not tweeting without archive urls for diff %s" % diff.id
 

test_diffengine.py

@@ -26,24 +26,27 @@
 )
 from diffengine.config import load_config
 from diffengine.text_builder import build_text
+from diffengine.utils import generate_config
 from exceptions.sendgrid import (
-    ConfigNotFoundError as SGConfigNotFoundError,
-    AlreadyEmailedError as SGAlreadyEmailedError,
-    ArchiveUrlNotFoundError as SGArchiveNotFoundError,
+    SendgridConfigNotFoundError,
+    AlreadyEmailedError,
+    SendgridArchiveUrlNotFoundError,
 )
 from exceptions.twitter import (
-    ConfigNotFoundError,
+    TwitterConfigNotFoundError,
     TokenNotFoundError,
     AlreadyTweetedError,
-    AchiveUrlNotFoundError,
+    TwitterAchiveUrlNotFoundError,
     UpdateStatusError,
 )
 
-if os.path.isdir("test"):
-    shutil.rmtree("test")
+test_home = "test"
+
+if os.path.isdir(test_home):
+    shutil.rmtree(test_home)
 
+generate_config(test_home, {"db": "sqlite:///:memory:"})
 # set things up but disable prompting for initial feed
-test_home = "test"
 init(test_home, prompt=False)
 
 # the sequence of these tests is significant
@@ -184,8 +187,7 @@ def test_config_file_integration(self):
         test_config = {
             "example": {"private_value": private_yaml_key, "public_value": public_value}
         }
-        config_file = os.path.join(test_home, "config.yaml")
-        yaml.dump(test_config, open(config_file, "w"), default_flow_style=False)
+        generate_config(test_home, test_config)
 
         # test!
         new_config = load_config(test_home)
@@ -365,13 +367,17 @@ def tearDown(self) -> None:
         logging.disable(logging.NOTSET)
 
     def test_raises_if_no_config_set(self):
-        self.assertRaises(ConfigNotFoundError, TwitterHandler, None, None)
-        self.assertRaises(ConfigNotFoundError, TwitterHandler, "myConsumerKey", None)
-        self.assertRaises(ConfigNotFoundError, TwitterHandler, None, "myConsumerSecret")
+        self.assertRaises(TwitterConfigNotFoundError, TwitterHandler, None, None)
+        self.assertRaises(
+            TwitterConfigNotFoundError, TwitterHandler, "myConsumerKey", None
+        )
+        self.assertRaises(
+            TwitterConfigNotFoundError, TwitterHandler, None, "myConsumerSecret"
+        )
 
         try:
             TwitterHandler("myConsumerKey", "myConsumerSecret")
-        except ConfigNotFoundError:
+        except TwitterConfigNotFoundError:
             self.fail("Twitter.__init__ raised ConfigNotFoundError unexpectedly!")
 
     def test_raises_if_no_token_provided(self):
@@ -399,15 +405,19 @@ def test_raises_if_not_all_archive_urls_are_present(self):
         }
 
         twitter = TwitterHandler("myConsumerKey", "myConsumerSecret")
-        self.assertRaises(AchiveUrlNotFoundError, twitter.tweet_diff, diff, token)
+        self.assertRaises(
+            TwitterAchiveUrlNotFoundError, twitter.tweet_diff, diff, token
+        )
 
         type(diff.old).archive_url = PropertyMock(return_value="http://test.url/old")
-        self.assertRaises(AchiveUrlNotFoundError, twitter.tweet_diff, diff, token)
+        self.assertRaises(
+            TwitterAchiveUrlNotFoundError, twitter.tweet_diff, diff, token
+        )
 
         type(diff.new).archive_url = PropertyMock(return_value="http://test.url/new")
         try:
             twitter.tweet_diff(diff, token)
-        except AchiveUrlNotFoundError:
+        except TwitterAchiveUrlNotFoundError:
             self.fail("twitter.tweet_diff raised AchiveUrlNotFoundError unexpectedly!")
 
     class MockedStatus(MagicMock):
@@ -558,10 +568,10 @@ def test_raises_if_no_config_set(self):
         type(diff).emailed = PropertyMock(return_value=False)
         sendgrid = SendgridHandler({})
 
-        self.assertRaises(SGConfigNotFoundError, sendgrid.publish_diff, diff, {})
+        self.assertRaises(SendgridConfigNotFoundError, sendgrid.publish_diff, diff, {})
         try:
             sendgrid.publish_diff(diff, self.config["sendgrid"])
-        except SGConfigNotFoundError:
+        except SendgridConfigNotFoundError:
             self.fail("sendgrid.publish_diff raised ConfigNotFoundError unexpectedly!")
 
     def test_raises_if_already_emailed(self):
@@ -570,26 +580,32 @@ def test_raises_if_already_emailed(self):
 
         sendgrid = SendgridHandler(self.config["sendgrid"])
         self.assertRaises(
-            SGAlreadyEmailedError, sendgrid.publish_diff, diff, self.config["sendgrid"]
+            AlreadyEmailedError, sendgrid.publish_diff, diff, self.config["sendgrid"]
         )
 
     def test_raises_if_not_all_archive_urls_are_present(self):
         diff = get_mocked_diff(False)
 
         sendgrid = SendgridHandler(self.config["sendgrid"])
         self.assertRaises(
-            SGArchiveNotFoundError, sendgrid.publish_diff, diff, self.config["sendgrid"]
+            SendgridArchiveUrlNotFoundError,
+            sendgrid.publish_diff,
+            diff,
+            self.config["sendgrid"],
         )
 
         type(diff.old).archive_url = PropertyMock(return_value="http://test.url/old")
         self.assertRaises(
-            SGArchiveNotFoundError, sendgrid.publish_diff, diff, self.config["sendgrid"]
+            SendgridArchiveUrlNotFoundError,
+            sendgrid.publish_diff,
+            diff,
+            self.config["sendgrid"],
         )
 
         type(diff.new).archive_url = PropertyMock(return_value="http://test.url/new")
         try:
             sendgrid.publish_diff(diff, self.config["sendgrid"])
-        except SGArchiveNotFoundError:
+        except SendgridArchiveUrlNotFoundError:
             self.fail(
                 "sendgrid.publish_diff raised AchiveUrlNotFoundError unexpectedly!"
             )