compute giving and receiving amounts more accurately

fixes #1127

branch.sql

@@ -1,5 +1,26 @@
 BEGIN;
     DROP VIEW goal_summary;
     ALTER TABLE tips ADD COLUMN is_funded boolean;
+
+    -- Needs to be recreated to include the new column
+    DROP VIEW current_tips;
+    CREATE VIEW current_tips AS
+        SELECT DISTINCT ON (tipper, tippee) *
+          FROM tips
+      ORDER BY tipper, tippee, mtime DESC;
+
+    -- Allow updating is_funding via the current_tips view for convenience
+    CREATE FUNCTION update_tip() RETURNS trigger AS $$
+        BEGIN
+            UPDATE tips
+               SET is_funded = NEW.is_funded
+             WHERE id = NEW.id;
+            RETURN NULL;
+        END;
+    $$ LANGUAGE plpgsql;
+
+    CREATE TRIGGER update_current_tip INSTEAD OF UPDATE ON current_tips
+        FOR EACH ROW EXECUTE PROCEDURE update_tip();
+
     \i fake_payday.sql
 END;

gratipay/billing/__init__.py

@@ -21,6 +21,8 @@
 import balanced
 from aspen.utils import typecheck
 
+from gratipay.models.participant import Participant
+
 
 def store_result(db, thing, username, new_result):
     """Update the participant's last_{ach,bill}_result in the DB.
@@ -50,21 +52,15 @@ def store_result(db, thing, username, new_result):
         return
     if p.is_suspicious or new_result == p.old_result:
         return
-    if new_result == '':
-        op = '+'
-    else:
-        op = '-'
-    db.run("""
-        UPDATE participants
-           SET receiving = (receiving {0} amount)
-             , npatrons = (npatrons {0} 1)
-          FROM ( SELECT DISTINCT ON (tippee) tippee, amount
-                   FROM tips
-                  WHERE tipper=%(tipper)s
-               ORDER BY tippee, mtime DESC
-               ) foo
-         WHERE tippee = username;
-    """.format(op), dict(tipper=username))
+    with db.get_cursor() as cursor:
+        Participant.from_username(username).update_giving(cursor)
+        tippees = cursor.all("""
+            SELECT tippee
+              FROM current_tips
+             WHERE tipper=%(tipper)s;
+        """, dict(tipper=username))
+        for tippee in tippees:
+            Participant.from_username(tippee).update_receiving(cursor)
 
 
 def get_balanced_account(db, username, balanced_customer_href):

gratipay/billing/payday.py

@@ -26,6 +26,10 @@
 from psycopg2 import IntegrityError
 
 
+with open('fake_payday.sql') as f:
+    FAKE_PAYDAY = f.read()
+
+
 def threaded_map(func, iterable, threads=5):
     pool = ThreadPool(threads)
     r = pool.map(func, iterable)
@@ -633,24 +637,8 @@ def update_stats(self):
 
 
     def update_receiving_amounts(self):
-        UPDATE = """
-            CREATE OR REPLACE TEMPORARY VIEW total_receiving AS
-                SELECT tippee, sum(amount) AS amount, count(*) AS ntippers
-                  FROM current_tips
-                  JOIN participants p ON p.username = tipper
-                 WHERE p.is_suspicious IS NOT TRUE
-                   AND p.last_bill_result = ''
-                   AND amount > 0
-              GROUP BY tippee;
-
-            UPDATE participants
-               SET receiving = (amount + taking)
-                 , npatrons = ntippers
-              FROM total_receiving
-             WHERE tippee = username;
-        """
         with self.db.get_cursor() as cursor:
-            cursor.execute(UPDATE)
+            cursor.execute(FAKE_PAYDAY)
         log("Updated receiving amounts.")
 
 

gratipay/models/participant.py

@@ -37,7 +37,6 @@
 from gratipay.models._mixin_team import MixinTeam
 from gratipay.models.account_elsewhere import AccountElsewhere
 from gratipay.utils.username import safely_reserve_a_username
-from gratipay import billing
 from gratipay.utils import is_card_expiring
 
 
@@ -581,40 +580,67 @@ def update_is_closed(self, is_closed, cursor=None):
             self.set_attributes(is_closed=is_closed)
 
     def update_giving(self, cursor=None):
-        giving = (cursor or self.db).one("""
+        # Update is_funded on tips
+        if self.last_bill_result == '':
+            (cursor or self.db).run("""
+                UPDATE current_tips
+                   SET is_funded = true
+                 WHERE tipper = %s
+                   AND is_funded IS NOT true
+            """, (self.username,))
+        else:
+            tips = (cursor or self.db).all("""
+                SELECT t.*
+                  FROM current_tips t
+                  JOIN participants p2 ON p2.username = t.tippee
+                 WHERE t.tipper = %s
+                   AND t.amount > 0
+                   AND p2.is_suspicious IS NOT true
+              ORDER BY p2.claimed_time IS NULL, t.ctime ASC
+            """, (self.username,))
+            fake_balance = self.balance + self.receiving - self.taking
+            for tip in tips:
+                if tip.amount > fake_balance:
+                    is_funded = False
+                else:
+                    fake_balance -= tip.amount
+                    is_funded = True
+                if tip.is_funded == is_funded:
+                    continue
+                (cursor or self.db).run("""
+                    UPDATE tips
+                       SET is_funded = %s
+                     WHERE id = %s
+                """, (is_funded, tip.id))
+
+        # Update giving and pledging on participant
+        giving, pledging = (cursor or self.db).one("""
+            WITH our_tips AS (
+                     SELECT amount, tippee, p2.claimed_time
+                       FROM current_tips
+                       JOIN participants p2 ON p2.username = tippee
+                      WHERE tipper = %(username)s
+                        AND p2.is_suspicious IS NOT true
+                        AND amount > 0
+                        AND is_funded
+                 )
             UPDATE participants p
                SET giving = COALESCE((
                        SELECT sum(amount)
-                         FROM current_tips
-                         JOIN participants p2 ON p2.username = tippee
-                        WHERE tipper = p.username
-                          AND p2.claimed_time IS NOT NULL
-                          AND p2.is_suspicious IS NOT true
-                     GROUP BY tipper
+                         FROM our_tips
+                        WHERE claimed_time IS NOT NULL
                    ), 0)
-             WHERE p.username = %s
-         RETURNING giving
-        """, (self.username,))
-        self.set_attributes(giving=giving)
-
-    def update_pledging(self, cursor=None):
-        pledging = (cursor or self.db).one("""
-            UPDATE participants p
-               SET pledging = COALESCE((
+                 , pledging = COALESCE((
                        SELECT sum(amount)
-                         FROM current_tips
-                         JOIN participants p2 ON p2.username = tippee
+                         FROM our_tips
                          JOIN elsewhere ON elsewhere.participant = tippee
-                        WHERE tipper = p.username
-                          AND p2.claimed_time IS NULL
+                        WHERE claimed_time IS NULL
                           AND elsewhere.is_locked = false
-                          AND p2.is_suspicious IS NOT true
-                     GROUP BY tipper
                    ), 0)
-             WHERE p.username = %s
-         RETURNING pledging
-        """, (self.username,))
-        self.set_attributes(pledging=pledging)
+             WHERE p.username = %(username)s
+         RETURNING giving, pledging
+        """, dict(username=self.username))
+        self.set_attributes(giving=giving, pledging=pledging)
 
     def update_receiving(self, cursor=None):
         if self.IS_PLURAL:
@@ -626,8 +652,8 @@ def update_receiving(self, cursor=None):
                        JOIN participants p2 ON p2.username = tipper
                       WHERE tippee = %(username)s
                         AND p2.is_suspicious IS NOT true
-                        AND p2.last_bill_result = ''
                         AND amount > 0
+                        AND is_funded
                  )
             UPDATE participants p
                SET receiving = (COALESCE((
@@ -710,10 +736,7 @@ def set_tip_to(self, tippee, amount, update_self=True, update_tippee=True, curso
 
         if update_self:
             # Update giving/pledging amount of tipper
-            if tippee.is_claimed:
-                self.update_giving(cursor)
-            else:
-                self.update_pledging(cursor)
+            self.update_giving(cursor)
         if update_tippee:
             # Update receiving amount of tippee
             tippee.update_receiving(cursor)
@@ -771,7 +794,7 @@ def get_tip_distribution(self):
                        FROM tips
                        JOIN participants p ON p.username = tipper
                       WHERE tippee=%s
-                        AND last_bill_result = ''
+                        AND is_funded
                         AND is_suspicious IS NOT true
                    ORDER BY tipper
                           , mtime DESC
@@ -1060,7 +1083,7 @@ def take_over(self, account, have_confirmation=False):
 
             -- Get all the latest tips from everyone to everyone.
 
-            SELECT ctime, tipper, tippee, amount
+            SELECT ctime, tipper, tippee, amount, is_funded
               FROM current_tips
              WHERE amount > 0;
 
@@ -1073,9 +1096,9 @@ def take_over(self, account, have_confirmation=False):
             -- dead and the live account, then we create one new combined tip
             -- to the live account (via the GROUP BY and sum()).
 
-            INSERT INTO tips (ctime, tipper, tippee, amount)
+            INSERT INTO tips (ctime, tipper, tippee, amount, is_funded)
 
-                 SELECT min(ctime), tipper, %(live)s AS tippee, sum(amount)
+                 SELECT min(ctime), tipper, %(live)s AS tippee, sum(amount), bool_and(is_funded)
 
                    FROM __temp_unique_tips
 
@@ -1311,9 +1334,10 @@ def take_over(self, account, have_confirmation=False):
             self.set_attributes(balance=new_balance)
 
         self.update_avatar()
-        self.update_giving()
-        self.update_pledging()
+
+        # Note: the order matters here, receiving needs to be updated before giving
         self.update_receiving()
+        self.update_giving()
 
     def delete_elsewhere(self, platform, user_id):
         """Deletes account elsewhere unless the user would not be able
@@ -1346,6 +1370,7 @@ def credit_card_expiring(self, request, response):
             return False
 
         try:
+            from gratipay import billing
             card = billing.BalancedCard(self.balanced_customer_href)
             year, month = card['expiration_year'], card['expiration_month']
             if not (year and month):

www/about/stats.spt

@@ -41,7 +41,7 @@ tips_stats = db.one("""
       JOIN participants p ON p.username = tipper
       JOIN participants p2 ON p2.username = tippee
      WHERE amount > 0
-       AND p.last_bill_result = ''
+       AND is_funded
        AND p2.claimed_time IS NOT NULL
        AND p.is_suspicious IS NOT true
        AND p2.is_suspicious IS NOT true
@@ -57,7 +57,7 @@ average_tippees = int(db.one("""\
                JOIN participants p ON p.username = tipper
                JOIN participants p2 on p2.username = tippee
               WHERE amount > 0
-                AND p.last_bill_result = ''
+                AND is_funded
                 AND p2.claimed_time IS NOT NULL
                 AND p.is_suspicious IS NOT true
                 AND p2.is_suspicious IS NOT true
@@ -82,7 +82,7 @@ TIP_DISTRIBUTION = """
       JOIN participants p ON p.username = tipper
       JOIN participants p2 on p2.username = tippee
      WHERE amount > 0
-       AND p.last_bill_result = ''
+       AND is_funded
        AND p2.claimed_time IS NOT NULL
        AND p.is_suspicious IS NOT true
        AND p2.is_suspicious IS NOT true

www/about/tip-distribution.json.spt

@@ -8,7 +8,7 @@ website.db.all("""
               FROM tips
               JOIN participants p ON p.username = tipper
               JOIN participants p2 on p2.username = tippee
-             WHERE p.last_bill_result = ''
+             WHERE is_funded
                AND p2.claimed_time IS NOT NULL
                AND NOT (p.is_suspicious IS true)
                AND NOT (p2.is_suspicious IS true)

www/for/%slug/index.html.spt

@@ -76,7 +76,6 @@ givers = query_cache.all("""
      WHERE is_suspicious IS NOT true
        AND giving > 0
        AND cc.is_member
-       AND last_bill_result = ''
   ORDER BY giving DESC
   LIMIT %s
   OFFSET %s