Merge branch 'master' into record-ref-in-masspay

.travis.yml

@@ -17,8 +17,6 @@ before_install:
  - sudo apt-get update
  - sudo apt-get install postgresql-9.6
 
- - npm install marky-markdown
-
  # For ttw tests
  - wget https://github.com/mozilla/geckodriver/releases/download/v0.15.0/geckodriver-v0.15.0-linux64.tar.gz
  - mkdir -p geckodriver
@@ -30,14 +28,21 @@ cache:
   directories:
     - env/bin
     - env/lib/python2.7/site-packages
-    - node_modules
     - geckodriver
 install:
  - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf env; fi
  - touch requirements.txt package.json
  - make env -j2
 before_script:
  - echo "DATABASE_URL=dbname=gratipay" | tee -a tests/local.env local.env
+
+ # Turn off all cron threads.
+ - echo "UPDATE_CTA_EVERY=0" >> local.env
+ - echo "CHECK_DB_EVERY=0" >> local.env
+ - echo "CHECK_NPM_SYNC_EVERY=0" >> local.env
+ - echo "EMAIL_QUEUE_FLUSH_EVERY=0" >> local.env
+ - echo "EMAIL_QUEUE_LOG_METRICS_EVERY=0" >> local.env
+
  - psql -U postgres -c 'CREATE DATABASE "gratipay";'
  - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rfv tests/py/fixtures; fi
 script: LD_LIBRARY_PATH=/usr/local/lib xvfb-run make bgrun test doc

README.md

@@ -496,11 +496,6 @@ an array of objects, one per week, showing aggregate numbers over time. The
 an array of objects, one per week, showing aggregate numbers over time. The old
 charts page used to use this.
 
-**[/about/stats.json](https://gratipay.com/about/stats.json)**
-([source](https://github.com/gratipay/gratipay.com/tree/master/www/about/stats.spt))&mdash;<i>public</i>&mdash;Returns
-an object giving a point-in-time snapshot of Gratipay. The
-[stats](https://gratipay.com/about/stats.html) page displays the same info.
-
 **/`~username`/public.json**
 ([example](https://gratipay.com/Gratipay/public.json),
 [source](https://github.com/gratipay/gratipay.com/blob/master/www/~/%25username/public.json.spt))&mdash;<i>public</i>&mdash;Returns an object with these keys:

defaults.env

@@ -76,6 +76,7 @@ OPENSTREETMAP_AUTH_URL=http://www.openstreetmap.org
 EMAIL_QUEUE_FLUSH_EVERY=60
 EMAIL_QUEUE_SLEEP_FOR=1
 EMAIL_QUEUE_ALLOW_UP_TO=3
+EMAIL_QUEUE_LOG_METRICS_EVERY=0
 
 UPDATE_CTA_EVERY=300
 CHECK_DB_EVERY=600

docs/doc.env

@@ -2,3 +2,10 @@ ASPEN_PROJECT_ROOT=..
 ASPEN_WWW_ROOT=../www
 WEBDRIVER_BROWSER='phantomjs'
 WEBDRIVER_BASE_URL='http://localhost:8537'
+
+# Turn off all cron threads.
+UPDATE_CTA_EVERY=0
+CHECK_DB_EVERY=0
+CHECK_NPM_SYNC_EVERY=0
+EMAIL_QUEUE_FLUSH_EVERY=0
+EMAIL_QUEUE_LOG_METRICS_EVERY=0

emails/approved-1.0.spt

@@ -1,4 +1,4 @@
-{{ _("1.0 Payout: Approved!") }}
+{{ _("1.0 payout approved!") }}
 [---] text/html
 {{ _("We've approved your application for a payout of your Gratipay 1.0 balance. Watch for that during our next weekly payment cycle.") }}
 <br>

emails/identity-viewed.spt

@@ -1,4 +1,4 @@
-{{ _("Identity Viewed") }}
+{{ _("Identity viewed") }}
 [---] text/html
 {{ _( "This is a transactional email to let you know that {a_viewer}{viewer}{_a} viewed your identity information for {a_country}{country_name}{_a} on Gratipay."
     , viewer=viewer

emails/project-review.spt

@@ -1,20 +1,19 @@
 {{ _("We have your application!") }}
 
 [---] text/html
-{{ _("Thanks for your project application for {0}! Your public project page is: "
-     "{1} "
-     "and your public review ticket is: "
-     "{2}. "
-     "You can watch and participate in our review process there. We will send a notification to {3} when we finish our review. "
-     "Thanks for applying!",
-     team_name, team_url, review_url, email) }}
+{{ _( "Thanks for applying to use Gratipay! The next step is for us to review your application, "
+      "which happens on {a}this public GitHub issue{_a}. You can watch and participate in our "
+      "review there, and we will email you again when we finish (usually in about a week). "
+      "Thanks again!"
+    , a='<a href="{}">'.format(review_url)|safe
+    , _a='</a>'|safe
+     ) }}
 
 [---] text/plain
-{{ _("Thanks for your project application for {0}! Your public project page is: "
-     "{1} "
-     "and your public review ticket is: "
-     "{2}. "
-     "You can watch and participate in our review process there. We will send a notification to {3} when we finish our review. "
-     "Thanks for applying!",
-     team_name, team_url, review_url, email) }}
+{{ _( "Thanks for applying to use Gratipay! The next step is for us to review your application, "
+      "which happens on this public GitHub issue:") }}
 
+{{ review_url }}
+
+{{ _( "You can watch and participate in our review there, and we will email you again when we "
+      "finish (usually in about a week). Thanks again!") }}

emails/rejected-1.0.spt

@@ -1,4 +1,4 @@
-{{ _("1.0 Payout: Rejected") }}
+{{ _("1.0 payout rejected") }}
 [---] text/html
 {{ _("We've rejected your application for a payout of your Gratipay 1.0 balance, because we couldn't determine what product or service you were offering for which people were paying you on Gratipay. Please reply if you would like to discuss this further.") }}
 [---] text/plain

emails/team-approved.spt

@@ -1,4 +1,4 @@
-{{ _("Project Application Approved!") }}
+{{ _("Project application approved!") }}
 [---] text/html
 {{ _( "We've approved your application for the '{team}' project on Gratipay. For details, please refer to {a}our review ticket{_a}."
     , team=team_name

emails/team-rejected.spt

@@ -1,4 +1,4 @@
-{{ _("Project Application Rejected") }}
+{{ _("Project application rejected") }}
 [---] text/html
 {{ _( "We've rejected your application for the '{team}' project on Gratipay. For details, please refer to {a}our review ticket{_a}."
     , team=team_name

gratipay/application.py

@@ -7,7 +7,7 @@
 from .cron import Cron
 from .models import GratipayDB
 from .payday_runner import PaydayRunner
-from .project_review_repo import ProjectReviewRepo
+from .project_review_process import ProjectReviewProcess
 from .website import Website
 
 
@@ -46,15 +46,16 @@ def __init__(self):
         self.install_periodic_jobs(website, env, db)
         self.website = website
         self.payday_runner = PaydayRunner(self)
-        self.project_review_repo = ProjectReviewRepo(env)
+        self.project_review_process = ProjectReviewProcess(env, db, self.email_queue)
 
 
     def install_periodic_jobs(self, website, env, db):
         cron = Cron(website)
-        cron(env.update_cta_every, lambda: utils.update_cta(website))
+        cron(env.update_cta_every, website.update_cta)
         cron(env.check_db_every, db.self_check, True)
-        cron(env.email_queue_flush_every, self.email_queue.flush, True)
         cron(env.check_npm_sync_every, lambda: sync_npm.check(db))
+        cron(env.email_queue_flush_every, self.email_queue.flush, True)
+        cron(env.email_queue_log_metrics_every, self.email_queue.log_metrics)
 
 
     def add_event(self, c, type, payload):

gratipay/cli/dequeue_emails.py

@@ -3,6 +3,7 @@
 
 import sys
 
+from aspen import log
 from gratipay.application import Application
 
 
@@ -15,4 +16,5 @@ def main(_argv=sys.argv, _input=raw_input):
     process/dyno we can start with this.
 
     """
+    log('Instantiating Application from gratipay.cli.dequeue_emails')
     Application().email_queue.flush()

gratipay/cli/payday.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from aspen import log
 from gratipay.application import Application
 
 
 def main():
     try:
+        log('Instantiating Application from gratipay.cli.payday')
         Application().payday_runner.run_payday()
     except KeyboardInterrupt:
         pass

gratipay/cli/queue_branch_email.py

@@ -4,6 +4,7 @@
 import sys
 import random
 
+from aspen import log
 from gratipay.application import Application
 
 
@@ -42,7 +43,10 @@ def main(_argv=sys.argv, _input=raw_input, _print=print, _app=None):
             - push to GitHub
 
     """
-    app = _app or Application()
+    if _app is None:
+        log('Instantiating Application from gratipay.cli.dequeue_emails')
+        _app = Application()
+    app = _app
 
     def prompt(msg):
         answer = _input(msg + " [y/N]")

gratipay/cron.py

@@ -2,7 +2,7 @@
 from time import sleep
 import traceback
 
-from aspen import log_dammit
+from aspen import log, log_dammit
 
 
 class Cron(object):
@@ -15,7 +15,10 @@ def __init__(self, website):
 
     def __call__(self, period, func, exclusive=False):
         if period <= 0:
+            log('Cron: not installing {}.'.format(func.__name__))
             return
+        log('Cron: installing {} to run every {} seconds{}.'.format(
+            func.__name__, period, ' with a lock' if exclusive else ''))
         if exclusive and not self.has_lock:
             self.exclusive_jobs.append((period, func))
             self._wait_for_lock()

gratipay/email.py

@@ -187,6 +187,12 @@ def render(t, context):
         return message
 
 
+    def log_metrics(self, _print=print):
+        ndead = self.db.one('SELECT COUNT(*) FROM email_queue WHERE dead')
+        ntotal = self.db.one('SELECT COUNT(*) FROM email_queue')
+        _print('count#email_queue_dead=%d count#email_queue_total=%d' % (ndead, ntotal))
+
+
     def purge(self):
         """Remove all messages from the queue.
         """

gratipay/fake_data.py

@@ -246,13 +246,16 @@ def random_country_id(db):
 
 
 def fake_package(db):
-    insert_fake_data( db
-                    , 'packages'
-                    , package_manager='npm'
-                    , name=faker.word()
-                    , description=fake_sentence(stop=50)
-                    , emails=[faker.email() for i in range(random.choice(range(10)))]
-                     )
+    try:
+        insert_fake_data( db
+                        , 'packages'
+                        , package_manager='npm'
+                        , name=faker.word()
+                        , description=fake_sentence(stop=50)
+                        , emails=[faker.email() for i in range(random.choice(range(10)))]
+                         )
+    except IntegrityError:
+        return fake_package(db)
 
 
 def prep_db(db):

gratipay/models/participant/packages.py

@@ -6,6 +6,61 @@
 
 class Packages(object):
 
+    def get_packages_for_claiming(self, manager):
+
+        """Return a list of packages on the named ``manager`` for which the
+        participant has verified an email address on Gratipay, along with the
+        current package owner on Gratipay (if any).
+
+        :param string manager: the name of the package manager on which to look
+            for potential packages
+
+        :return: a list of (:py:class:`~gratipay.models.package.Package`,
+            :py:class:`Participant`, is_primary, email_address) tuples, where the
+            participant is the one who has already claimed the package (or ``None``),
+            and the email address is the single best match (primary, then
+            alphabetically first from among non-primary verified addresses)
+
+        """
+        return self.db.all('''
+
+            WITH verified_emails AS (
+                SELECT e.address
+                     , e.address = p.email_address is_primary
+                  FROM emails e
+             LEFT JOIN participants p
+                    ON p.id = e.participant_id
+                 WHERE e.participant_id=%s
+                   AND e.verified is true
+            )
+            SELECT pkg.*::packages                  package
+                 , p.*::participants                claimed_by
+                 , (SELECT is_primary
+                      FROM verified_emails
+                     WHERE address = ANY(emails)
+                  ORDER BY is_primary DESC, address
+                     LIMIT 1)                       email_address_is_primary
+                 , (SELECT address
+                      FROM verified_emails
+                     WHERE address = ANY(emails)
+                  ORDER BY is_primary DESC, address
+                     LIMIT 1)                       email_address
+              FROM packages pkg
+         LEFT JOIN teams_to_packages tp
+                ON pkg.id = tp.package_id
+         LEFT JOIN teams t
+                ON t.id = tp.team_id
+         LEFT JOIN participants p
+                ON t.owner = p.username
+             WHERE package_manager=%s
+               AND pkg.emails && array(SELECT address FROM verified_emails)
+          ORDER BY email_address_is_primary DESC
+                 , email_address ASC
+                 , pkg.name ASC
+
+        ''', (self.id, manager))
+
+
     def start_package_claims(self, c, nonce, *packages):
         """Takes a cursor, nonce and list of packages, inserts into ``claims``
         and returns ``None`` (or raise :py:exc:`NoPackages`).
@@ -57,7 +112,7 @@ def finish_package_claims(self, cursor, nonce, *packages):
             team = package.get_or_create_linked_team(cursor, self)
             teams.append(team)
             team_ids.append(team.id)
-        review_url = self.app.project_review_repo.create_issue(*teams)
+        review_url = self.app.project_review_process.start(*teams)
 
         cursor.run('DELETE FROM claims WHERE nonce=%s', (nonce,))
         cursor.run('UPDATE teams SET review_url=%s WHERE id=ANY(%s)', (review_url, team_ids,))

gratipay/models/team/__init__.py

@@ -249,11 +249,6 @@ def get_upcoming_payment(self):
         """, {'team_id': self.id, 'mcharge': MINIMUM_CHARGE})
 
 
-    def set_review_url(self, review_url):
-        self.db.run("UPDATE teams SET review_url=%s WHERE id=%s", (review_url, self.id))
-        self.set_attributes(review_url=review_url)
-
-
     def get_og_title(self):
         out = self.name
         receiving = self.receiving