[twitter] fix several errors (#2212, #2216, #2225)

- fix Tweets with deleted quotes - fix suspended Tweets without 'legacy' entry - fix unified_cards without 'type'
mikf · Jan 25, 2022 · 2bf554a · 2bf554a
1 parent fbd1754
commit 2bf554a
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,7 @@
 # Changelog
 
+## Unreleased
+
 ## 1.20.2 - 2022-01-24
 ### Additions
 - [twitter] add `event` extractor (closes [#2109](https://github.com/mikf/gallery-dl/issues/2109))

diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
@@ -180,7 +180,7 @@ def _extract_card(self, tweet, files):
             else:
                 bval = bvals["unified_card"]["string_value"]
             data = json.loads(bval)
-            if data["type"] == "image_carousel_website":
+            if data.get("type") == "image_carousel_website":
                 self._extract_media(
                     tweet, data["media_entities"].values(), files)
                 return
@@ -383,6 +383,10 @@ class TwitterTimelineExtractor(TwitterExtractor):
             "range": "1-40",
             "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40",
         }),
+        # suspended account (#2216)
+        ("https://twitter.com/realDonaldTrump", {
+            "exception": exception.NotFoundError,
+        }),
         ("https://mobile.twitter.com/supernaturepics?p=i"),
         ("https://www.twitter.com/id:2976459548"),
         ("https://twitter.com/i/user/2976459548"),
@@ -612,6 +616,10 @@ class TwitterTweetExtractor(TwitterExtractor):
             "pattern": r"https://pbs\.twimg\.com/media/F.+=png",
             "count": 6,
         }),
+        # unified_card without type
+        ("https://twitter.com/i/web/status/1466183847628865544", {
+            "count": 0,
+        }),
         # original retweets (#1026)
         ("https://twitter.com/jessica_3978/status/1296304589591810048", {
             "options": (("retweets", "original"),),
@@ -632,6 +640,10 @@ class TwitterTweetExtractor(TwitterExtractor):
             "options": (("retweets", True),),
             "count": 4,
         }),
+        # deleted quote tweet (#2225)
+        ("https://twitter.com/i/web/status/1460044411165888515", {
+            "count": 0,
+        }),
     )
 
     def __init__(self, match):
@@ -1103,17 +1115,17 @@ def _pagination_tweets(self, endpoint, variables, path=None):
                         tweet = True
                     cursor = cursor.get("value")
 
-            for tweet in tweets:
+            for entry in tweets:
                 try:
-                    tweet = ((tweet.get("content") or tweet["item"])
+                    tweet = ((entry.get("content") or entry["item"])
                              ["itemContent"]["tweet_results"]["result"])
+                    legacy = tweet["legacy"]
                 except KeyError:
                     self.extractor.log.debug(
                         "Skipping %s (deleted)",
-                        tweet["entryId"].rpartition("-")[2])
+                        (entry.get("entryId") or "").rpartition("-")[2])
                     continue
 
-                legacy = tweet["legacy"]
                 if "retweeted_status_result" in legacy:
                     retweet = legacy["retweeted_status_result"]["result"]
                     if original_retweets:
@@ -1135,12 +1147,18 @@ def _pagination_tweets(self, endpoint, variables, path=None):
                 yield tweet
 
                 if "quoted_status_result" in tweet:
-                    quoted = tweet["quoted_status_result"]["result"]
-                    quoted["legacy"]["author"] = \
-                        quoted["core"]["user_results"]["result"]
-                    quoted["core"] = tweet["core"]
-                    quoted["legacy"]["quoted_by_id_str"] = tweet["rest_id"]
-                    yield quoted
+                    try:
+                        quoted = tweet["quoted_status_result"]["result"]
+                        quoted["legacy"]["author"] = \
+                            quoted["core"]["user_results"]["result"]
+                        quoted["core"] = tweet["core"]
+                        quoted["legacy"]["quoted_by_id_str"] = tweet["rest_id"]
+                        yield quoted
+                    except KeyError:
+                        self.extractor.log.debug(
+                            "Skipping quote of %s (deleted)",
+                            tweet.get("rest_id"))
+                        continue
 
             if not tweet or not cursor:
                 return

diff --git a/gallery_dl/version.py b/gallery_dl/version.py
@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.20.2"
+__version__ = "1.20.3-dev"