importer: Allow re-importing RGL matches

The original import of RGL matches got the teams swapped around in a few cases. Allow re-importing to fix these errors. Signed-off-by: Sean Anderson <[email protected]>
Forty-Bot · Jan 22, 2024 · b0ddf62 · b0ddf62
1 parent fe2dcbc
commit b0ddf62
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 23 deletions.
diff --git a/trends/importer/league.py b/trends/importer/league.py
@@ -300,23 +300,35 @@ def import_match(c, m):
         WHERE league = %(league)s
             AND compid = %(compid)s;""", m)
     c.execute("INSERT INTO map (map) SELECT unnest(%(maps)s::TEXT[]) ON CONFLICT DO NOTHING;", m)
-    c.execute(
-        """INSERT INTO match (
-               league, matchid, compid, divid, teamid1, teamid2, round_seq, scheduled, submitted,
-               mapids, score1, score2, forfeit, fetched
-           ) VALUES (
-               %(league)s, %(matchid)s, %(compid)s, %(divid)s, %(teamid1)s, %(teamid2)s, %(seq)s,
-               %(scheduled)s, %(submitted)s,
-               (SELECT
-                       coalesce(array_agg(mapid ORDER BY mapid), array[]::INT[])
-                   FROM map
-                   WHERE map = any(%(maps)s)
-               ), %(score1)s, %(score2)s, %(forfeit)s, %(fetched)s
-           ) ON CONFLICT (league, matchid)
-           DO UPDATE SET
-               scheduled = EXCLUDED.scheduled,
-               mapids = EXCLUDED.mapids,
-               score1 = EXCLUDED.score1,
-               score2 = EXCLUDED.score2,
-               forfeit = EXCLUDED.forfeit,
-               fetched = greatest(match.fetched, EXCLUDED.fetched);""", m)
+    # Can't use ON CONFLICT here because there are multiple unique indices which could match
+    try:
+        c.execute("SAVEPOINT match_update;")
+        c.execute(
+            """INSERT INTO match (
+                   league, matchid, compid, divid, teamid1, teamid2, round_seq, scheduled, submitted,
+                   mapids, score1, score2, forfeit, fetched
+               ) VALUES (
+                   %(league)s, %(matchid)s, %(compid)s, %(divid)s, %(teamid1)s, %(teamid2)s, %(seq)s,
+                   %(scheduled)s, %(submitted)s,
+                   (SELECT
+                           coalesce(array_agg(mapid ORDER BY mapid), array[]::INT[])
+                       FROM map
+                       WHERE map = any(%(maps)s)
+                   ), %(score1)s, %(score2)s, %(forfeit)s, %(fetched)s
+               );""", m)
+    except psycopg2.errors.UniqueViolation:
+        c.execute("ROLLBACK TO SAVEPOINT match_update;")
+        c.execute(
+            """UPDATE match SET
+                   scheduled = %(scheduled)s,
+                   mapids = (SELECT
+                           coalesce(array_agg(mapid ORDER BY mapid), array[]::INT[])
+                       FROM map
+                       WHERE map = any(%(maps)s)
+                   ),
+                   score1 = %(score1)s,
+                   score2 = %(score2)s,
+                   forfeit = %(forfeit)s,
+                   fetched = greatest(fetched, %(fetched)s)
+                WHERE league = %(league)s
+                    AND matchid = %(matchid)s;""", m)
diff --git a/trends/importer/rgl.py b/trends/importer/rgl.py
@@ -29,6 +29,9 @@ def filter_matchids(c, matchids):
             for row in cur:
                 yield row[0]
 
+def no_filter_matchids(c, matchids):
+    yield from matchids
+
 rgl_format_map = {
     'Sixes': 'sixes',
     'NR Sixes': 'sixes',
@@ -138,6 +141,8 @@ def parse_team(team):
 def create_rgl_parser(sub):
     rgl = sub.add_parser("rgl", help="Import rgl matches")
     rgl.set_defaults(importer=import_rgl_cli)
+    rgl.add_argument("-R", "--reimport", action='store_true',
+                     help="Reimport all matches, even if they are already present")
     rgl_sub = rgl.add_subparsers()
     f = rgl_sub.add_parser("file", help="Import from the local filesystem")
     f.set_defaults(fetcher=RGLFileFetcher)
@@ -164,16 +169,17 @@ def import_rgl_cli(args, c):
                        FROM match
                        WHERE league = 'rgl';""");
                 args.since = datetime.fromtimestamp(cur.fetchone()[0])
-        return import_rgl(c, args.fetcher(**vars(args)))
+        return import_rgl(c, args.fetcher(**vars(args)),
+                          no_filter_matchids if args.reimport else filter_matchids)
 
-def import_rgl(c, fetcher):
+def import_rgl(c, fetcher, filter=filter_matchids):
     @functools.cache
     def get_season(seasonid):
         return parse_season(fetcher.get_season(seasonid))
 
     cur = c.cursor()
     count = 0
-    for matchid in filter_matchids(c, fetcher.get_matchids()):
+    for matchid in filter(c, fetcher.get_matchids()):
         try:
             result = fetcher.get_match(matchid)
             res = parse_match(result)