Skip to content

Commit

Permalink
[twitter] simplify
Browse files Browse the repository at this point in the history 
- use dict with common GraphQL variables
- reduce 'variables' size with custom JSON encoder instance
- centralise TwitterAPI() creation
  • Loading branch information
@mikf
mikf committed Jan 23, 2022
1 parent 7cb2922 commit 729b07c
Showing 1 changed file with 34 additions and 111 deletions.
145 changes: 34 additions & 111 deletions gallery_dl/extractor/twitter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ def _init_sizes(self):

def items(self):
self.login()
self.api = TwitterAPI(self)
metadata = self.metadata()

for tweet in self.tweets():
Expand Down Expand Up @@ -395,7 +396,7 @@ def __init__(self, match):
self.user = "id:" + user_id

def tweets(self):
return TwitterAPI(self).user_tweets(self.user)
return self.api.user_tweets(self.user)


class TwitterRepliesExtractor(TwitterExtractor):
Expand All @@ -412,7 +413,7 @@ class TwitterRepliesExtractor(TwitterExtractor):
)

def tweets(self):
return TwitterAPI(self).user_tweets_and_replies(self.user)
return self.api.user_tweets_and_replies(self.user)


class TwitterMediaExtractor(TwitterExtractor):
Expand All @@ -429,7 +430,7 @@ class TwitterMediaExtractor(TwitterExtractor):
)

def tweets(self):
return TwitterAPI(self).user_media(self.user)
return self.api.user_media(self.user)


class TwitterLikesExtractor(TwitterExtractor):
Expand All @@ -442,7 +443,7 @@ def metadata(self):
return {"user_likes": self.user}

def tweets(self):
return TwitterAPI(self).user_likes(self.user)
return self.api.user_likes(self.user)


class TwitterBookmarkExtractor(TwitterExtractor):
Expand All @@ -452,7 +453,7 @@ class TwitterBookmarkExtractor(TwitterExtractor):
test = ("https://twitter.com/i/bookmarks",)

def tweets(self):
return TwitterAPI(self).user_bookmarks()
return self.api.user_bookmarks()


class TwitterListExtractor(TwitterExtractor):
Expand All @@ -466,7 +467,7 @@ class TwitterListExtractor(TwitterExtractor):
})

def tweets(self):
return TwitterAPI(self).list_latest_tweets_timeline(self.user)
return self.api.list_latest_tweets_timeline(self.user)


class TwitterListMembersExtractor(TwitterExtractor):
Expand Down Expand Up @@ -508,7 +509,7 @@ def metadata(self):
return {"search": text.unquote(self.user)}

def tweets(self):
return TwitterAPI(self).search_adaptive(text.unquote(self.user))
return self.api.search_adaptive(text.unquote(self.user))


class TwitterEventExtractor(TwitterExtractor):
Expand All @@ -521,7 +522,6 @@ class TwitterEventExtractor(TwitterExtractor):
})

def metadata(self):
self.api = TwitterAPI(self)
return {"event": self.api.live_event(self.user)}

def tweets(self):
Expand Down Expand Up @@ -638,11 +638,11 @@ def __init__(self, match):

def tweets(self):
if self.config("conversations", False):
return TwitterAPI(self).tweet_detail(self.tweet_id)
return self.api.tweet_detail(self.tweet_id)

tweets = []
tweet_id = self.tweet_id
for tweet in TwitterAPI(self).tweet_detail(tweet_id):
for tweet in self.api.tweet_detail(tweet_id):
if tweet["rest_id"] == tweet_id or \
tweet.get("_retweet_id_str") == tweet_id:
tweets.append(tweet)
Expand Down Expand Up @@ -730,6 +730,22 @@ def __init__(self, extractor):
"ext": "mediaStats,highlightedLabel,hasNftAvatar,"
"voiceInfo,superFollowMetadata",
}
self.variables = {
"includePromotedContent": False,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withClientEventToken": False,
"withBirdwatchNotes": False,
"withVoice": True,
"withV2Timeline": False,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
self._json_dumps = json.JSONEncoder(separators=(",", ":")).encode

cookies = extractor.session.cookies
cookiedomain = extractor.cookiedomain
Expand All @@ -755,20 +771,9 @@ def tweet_detail(self, tweet_id):
variables = {
"focalTweetId": tweet_id,
"with_rux_injections": False,
"includePromotedContent": True,
"withCommunity": True,
"withQuickPromoteEligibilityTweetFields": True,
"withBirdwatchNotes": False,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withVoice": True,
"withV2Timeline": False,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False
}
return self._pagination_tweets(
endpoint, variables, ("threaded_conversation_with_injections",))
Expand All @@ -778,18 +783,7 @@ def user_tweets(self, screen_name):
variables = {
"userId": self._user_id_by_screen_name(screen_name),
"count": 100,
"includePromotedContent": True,
"withQuickPromoteEligibilityTweetFields": True,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withVoice": True,
"withV2Timeline": False,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_tweets(endpoint, variables)

Expand All @@ -798,19 +792,7 @@ def user_tweets_and_replies(self, screen_name):
variables = {
"userId": self._user_id_by_screen_name(screen_name),
"count": 100,
"cursor": "HBaWwLnhx93RkygAAA==",
"includePromotedContent": True,
"withCommunity": True,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withVoice": True,
"withV2Timeline": False,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_tweets(endpoint, variables)

Expand All @@ -819,19 +801,6 @@ def user_media(self, screen_name):
variables = {
"userId": self._user_id_by_screen_name(screen_name),
"count": 100,
"includePromotedContent": False,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withClientEventToken": False,
"withBirdwatchNotes": False,
"withVoice": True,
"withV2Timeline": False,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_tweets(endpoint, variables)

Expand All @@ -840,35 +809,13 @@ def user_likes(self, screen_name):
variables = {
"userId": self._user_id_by_screen_name(screen_name),
"count": 100,
"includePromotedContent": False,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withClientEventToken": False,
"withBirdwatchNotes": False,
"withVoice": True,
"withV2Timeline": False,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_tweets(endpoint, variables)

def user_bookmarks(self):
endpoint = "/graphql/WgbaxqmzjFP7oxkh_PkW4g/Bookmarks"
variables = {
"count": 100,
"includePromotedContent": True,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_tweets(
endpoint, variables, ("bookmark_timeline", "timeline"))
Expand All @@ -878,14 +825,6 @@ def list_latest_tweets_timeline(self, list_id):
variables = {
"listId": list_id,
"count": 100,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_tweets(
endpoint, variables, ("list", "tweets_timeline", "timeline"))
Expand Down Expand Up @@ -918,7 +857,7 @@ def live_event(self, event_id):

def list_by_rest_id(self, list_id):
endpoint = "/graphql/BWEhzAk7k8TwbU4lKH2dpw/ListByRestId"
params = {"variables": json.dumps({
params = {"variables": self._json_dumps({
"listId": list_id,
"withSuperFollowsUserFields": True,
})}
Expand All @@ -932,15 +871,7 @@ def list_members(self, list_id):
variables = {
"listId": list_id,
"count": 100,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"withSafetyModeUserFields": True,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}
return self._pagination_users(
endpoint, variables, ("list", "members_timeline", "timeline"))
Expand All @@ -950,27 +881,16 @@ def user_following(self, screen_name):
variables = {
"userId": self._user_id_by_screen_name(screen_name),
"count": 100,
"includePromotedContent": False,
"withSuperFollowsUserFields": True,
"withBirdwatchPivots": False,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": True,
"__fs_interactive_text": False,
"__fs_dont_mention_me_view_api_enabled": False,
}

return self._pagination_users(endpoint, variables)

def user_by_screen_name(self, screen_name):
endpoint = "/graphql/7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
variables = {
params = {"variables": self._json_dumps({
"screen_name": screen_name,
"withSafetyModeUserFields": True,
"withSuperFollowsUserFields": True,
}
params = {"variables": json.dumps(variables)}
})}
try:
return self._call(endpoint, params)["data"]["user"]["result"]
except KeyError:
Expand Down Expand Up @@ -1128,13 +1048,14 @@ def _pagination_legacy(self, endpoint, params):
params["cursor"] = cursor

def _pagination_tweets(self, endpoint, variables, path=None):
variables.update(self.variables)
original_retweets = (self.extractor.retweets == "original")
pinned_tweet = self.extractor.pinned

while True:
tweets = []
cursor = tweet = stop = None
params = {"variables": json.dumps(variables)}
params = {"variables": self._json_dumps(variables)}
data = self._call(endpoint, params)["data"]

try:
Expand Down Expand Up @@ -1214,9 +1135,11 @@ def _pagination_tweets(self, endpoint, variables, path=None):
variables["cursor"] = cursor

def _pagination_users(self, endpoint, variables, path=None):
variables.update(self.variables)

while True:
cursor = entry = stop = None
params = {"variables": json.dumps(variables)}
params = {"variables": self._json_dumps(variables)}
data = self._call(endpoint, params)["data"]

try:
Expand Down

0 comments on commit 729b07c

Please sign in to comment.