diff --git a/argparser.py b/argparser.py index 22cccf4c..92c3e5a7 100644 --- a/argparser.py +++ b/argparser.py @@ -18,8 +18,7 @@ def parse_arguments() -> argparse.Namespace: argparser.add_argument( "--server", required=False, - help="Required: The name of \ - your server (e.g. `mstdn.thms.uk`)", + help="Required: The name of your server (e.g. `mstdn.thms.uk`)", ) argparser.add_argument( "--access-token", @@ -148,25 +147,22 @@ def parse_arguments() -> argparse.Namespace: "--on-done", required=False, default=None, - help="Provide \ - a url that will be pinged when processing has completed. You can use this for " - "'dead man switch' monitoring of your task", + help="Provide a url that will be pinged when processing has completed. You can " + "use this for 'dead man switch' monitoring of your task", ) argparser.add_argument( "--on-start", required=False, default=None, - help="Provide \ - a url that will be pinged when processing is starting. You can use this for " - "'dead man switch' monitoring of your task", + help="Provide a url that will be pinged when processing is starting. You can " + "use this for 'dead man switch' monitoring of your task", ) argparser.add_argument( "--on-fail", required=False, default=None, - help="Provide \ - a url that will be pinged when processing has failed. You can use this for " - "'dead man switch' monitoring of your task", + help="Provide a url that will be pinged when processing has failed. You can " + "use this for 'dead man switch' monitoring of your task", ) argparser.add_argument( "--log-level", diff --git a/fedifetcher/api/client.py b/fedifetcher/api/client.py index e2f3143f..1108ea0f 100644 --- a/fedifetcher/api/client.py +++ b/fedifetcher/api/client.py @@ -150,8 +150,8 @@ def handle_response_lists( if isinstance(body, list): body = {"list": body} if not isinstance(body, dict): - msg = f"Error with API on server {self.api_base_url}. \ -The server returned an unexpected response: {body}" + msg = (f"Error with API on server {self.api_base_url}. " + f"The server returned an unexpected response: {body}") raise TypeError( msg, ) diff --git a/fedifetcher/api/lemmy/api_lemmy.py b/fedifetcher/api/lemmy/api_lemmy.py index 09e2d139..568f55af 100644 --- a/fedifetcher/api/lemmy/api_lemmy.py +++ b/fedifetcher/api/lemmy/api_lemmy.py @@ -9,7 +9,7 @@ def get_user_posts_from_url( - parsed_url : tuple[str, str], + parsed_url: tuple[str, str], ) -> list[dict[str, str]] | None: """Get a list of posts from a user.""" logging.info(f"Getting posts from {parsed_url[0]} for user {parsed_url[1]}") @@ -26,21 +26,23 @@ def get_user_posts_from_url( return all_posts except requests.HTTPError: logging.error( -f"Error getting user posts for user {parsed_url[1]}: {response.text}") + f"Error getting user posts for user {parsed_url[1]}: {response.text}", + ) except requests.JSONDecodeError: logging.error( -f"Error decoding JSON for user {parsed_url[1]}. \ -{parsed_url[0]} may not be a Lemmy instance.") + f"Error decoding JSON for user {parsed_url[1]}. " + f"{parsed_url[0]} may not be a Lemmy instance.", + ) except Exception: - logging.exception( -f"Error getting user posts from {url}") + logging.exception(f"Error getting user posts from {url}") return None + def get_community_posts_from_url( - parsed_url : tuple[str, str]) -> list[dict[str, str]] | None: + parsed_url: tuple[str, str], +) -> list[dict[str, str]] | None: """Get a list of posts from a community.""" - logging.info( - f"Getting posts from {parsed_url[0]} for community {parsed_url[1]}") + logging.info(f"Getting posts from {parsed_url[0]} for community {parsed_url[1]}") try: url = f"https://{parsed_url[0]}/api/v3/post/list?community_name={parsed_url[1]}&sort=New&limit=50" response = helpers.get(url) @@ -50,16 +52,16 @@ def get_community_posts_from_url( post["url"] = post["ap_id"] return posts except requests.exceptions.Timeout: - logging.error( -f"Timeout getting posts for community {parsed_url[1]}") + logging.error(f"Timeout getting posts for community {parsed_url[1]}") except requests.exceptions.RequestException: - logging.exception( -f"Error getting posts for community {parsed_url[1]}") + logging.exception(f"Error getting posts for community {parsed_url[1]}") else: logging.error( -f"Error getting posts for community {parsed_url[1]}: {response.text}") + f"Error getting posts for community {parsed_url[1]}: {response.text}", + ) return None + def get_comment_context( server: str, toot_id: str, @@ -83,7 +85,8 @@ def get_comment_context( resp = helpers.get(comment) except Exception as ex: logging.error( -f"Error getting comment {toot_id} from {toot_url}. Exception: {ex}") + f"Error getting comment {toot_id} from {toot_url}. Exception: {ex}", + ) return [] if resp.status_code == helpers.Response.OK: try: @@ -92,24 +95,29 @@ def get_comment_context( return get_comments_urls(server, post_id, toot_url) except Exception as ex: logging.error( -f"Error parsing context for comment {toot_url}. Exception: {ex}") + f"Error parsing context for comment {toot_url}. Exception: {ex}", + ) return [] if resp.status_code == helpers.Response.TOO_MANY_REQUESTS: - reset = datetime.strptime(resp.headers["x-ratelimit-reset"], - "%Y-%m-%dT%H:%M:%S.%fZ").astimezone(UTC) + reset = datetime.strptime( + resp.headers["x-ratelimit-reset"], + "%Y-%m-%dT%H:%M:%S.%fZ", + ).astimezone(UTC) logging.warning( -f"Rate Limit hit when getting context for {toot_url}. \ -Waiting to retry at {resp.headers['x-ratelimit-reset']}") + f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at " + f"{resp.headers['x-ratelimit-reset']}", + ) time.sleep((reset - datetime.now(UTC)).total_seconds() + 1) return get_comment_context(server, toot_id, toot_url) return [] + def get_comments_urls( # noqa: PLR0912 - server : str, - post_id : str, - toot_url : str, - ) -> list[str]: + server: str, + post_id: str, + toot_url: str, +) -> list[str]: """Get the URLs of the comments of the given post. Args: @@ -138,37 +146,47 @@ def get_comments_urls( # noqa: PLR0912 return [] urls.append(res["post_view"]["post"]["ap_id"]) except Exception as ex: - logging.error(f"Error parsing post {post_id} from {toot_url}. \ - Exception: {ex}") + logging.error( + f"Error parsing post {post_id} from {toot_url}. Exception: {ex}", + ) url = f"https://{server}/api/v3/comment/list?post_id={post_id}&sort=New&limit=50" try: resp = helpers.get(url) except Exception as ex: - logging.error(f"Error getting comments for post {post_id} from {toot_url}. \ -Exception: {ex}") + logging.error( + f"Error getting comments for post {post_id} from {toot_url}. Exception: " + f"{ex}", + ) return [] if resp.status_code == helpers.Response.OK: try: res = resp.json() - list_of_urls = \ - [comment_info["comment"]["ap_id"] for comment_info in res["comments"]] + list_of_urls = [ + comment_info["comment"]["ap_id"] for comment_info in res["comments"] + ] logging.info(f"Got {len(list_of_urls)} comments for post {toot_url}") urls.extend(list_of_urls) except Exception as ex: logging.error( -f"Error parsing comments for post {toot_url}. Exception: {ex}") + f"Error parsing comments for post {toot_url}. Exception: {ex}", + ) else: return urls elif resp.status_code == helpers.Response.TOO_MANY_REQUESTS: - reset = datetime.strptime(resp.headers["x-ratelimit-reset"], - "%Y-%m-%dT%H:%M:%S.%fZ").astimezone(UTC) - logging.info(f"Rate Limit hit when getting comments for {toot_url}. Waiting to \ - retry at {resp.headers['x-ratelimit-reset']}") + reset = datetime.strptime( + resp.headers["x-ratelimit-reset"], + "%Y-%m-%dT%H:%M:%S.%fZ", + ).astimezone(UTC) + logging.info( + f"Rate Limit hit when getting comments for {toot_url}. Waiting to retry at " + f"{resp.headers['x-ratelimit-reset']}", + ) time.sleep((reset - datetime.now(UTC)).total_seconds() + 1) return get_comments_urls(server, post_id, toot_url) logging.error( -f"Error getting comments for post {toot_url}. Status code: {resp.status_code}") + f"Error getting comments for post {toot_url}. Status code: {resp.status_code}", + ) return [] diff --git a/fedifetcher/api/mastodon/api_mastodon.py b/fedifetcher/api/mastodon/api_mastodon.py index 57522d1e..491c0530 100644 --- a/fedifetcher/api/mastodon/api_mastodon.py +++ b/fedifetcher/api/mastodon/api_mastodon.py @@ -630,8 +630,8 @@ async def _get_trending_posts( ) return [] logging.info( - f"Got {len(got_trending_posts)} trending posts for \ - {self.client.api_base_url}", + f"Got {len(got_trending_posts)} trending posts for " + f"{self.client.api_base_url}", ) trending_posts: list[dict[str, str]] = [] trending_posts.extend(got_trending_posts) diff --git a/fedifetcher/api/postgresql/postgresql.py b/fedifetcher/api/postgresql/postgresql.py index c36512db..f09d09c8 100644 --- a/fedifetcher/api/postgresql/postgresql.py +++ b/fedifetcher/api/postgresql/postgresql.py @@ -284,8 +284,8 @@ def get_from_cache(self, url: str) -> Status | None: columns = [column[0] for column in cursor.description] result = dict(zip(columns, result, strict=False)) logging.info( - f"Got status from cache: {url} \ -Original: {result.get('original')}, ID: {result.get('status_id')}", + f"Got status from cache: {url} Original: " + f"{result.get('original')}, ID: {result.get('status_id')}", ) status = Status( id=result.get("status_id"), diff --git a/fedifetcher/find_context.py b/fedifetcher/find_context.py index f44004e2..0482eebc 100644 --- a/fedifetcher/find_context.py +++ b/fedifetcher/find_context.py @@ -137,6 +137,6 @@ async def add_context_urls_wrapper( logging.warning(f"Failed {result}") logging.info( - f"\033[1mAdded {count} new statuses (with {failed} failures, \ -{already_added} already seen)\033[0m", + f"\033[1mAdded {count} new statuses (with {failed} failures, {already_added} " + f"already seen)\033[0m", ) diff --git a/fedifetcher/find_trending_posts.py b/fedifetcher/find_trending_posts.py index 9607c837..b8419b26 100644 --- a/fedifetcher/find_trending_posts.py +++ b/fedifetcher/find_trending_posts.py @@ -57,9 +57,9 @@ def add_post_to_dict( ) if original: logging.info( - f"Reblogs: {trending_post['reblogs_count']} \ -Favourites: {trending_post['favourites_count']} \ -From: {t_post_url}", + f"Reblogs: {trending_post['reblogs_count']} " + f"Favourites: {trending_post['favourites_count']} " + f"From: {t_post_url}", ) trending_posts_dict[t_post_url] = trending_post trending_posts_dict[t_post_url]["original"] = "Yes" @@ -67,18 +67,18 @@ def add_post_to_dict( if t_post_url in trending_posts_dict: if "original" not in trending_posts_dict[t_post_url]: logging.debug( - f"\033[3;33mReblogs: {trending_post['reblogs_count']} \ -Favourites: {trending_post['favourites_count']} \ -Copy: {t_post_url}\033[0m", + f"\033[3;33mReblogs: {trending_post['reblogs_count']} " + f"Favourites: {trending_post['favourites_count']} " + f"Copy: {t_post_url}\033[0m", ) increment_count(t_post_url, trending_post, trending_posts_dict) return False logging.debug(f"Already seen {t_post_url} from origin") return True # We already have the original logging.debug( - f"\033[3;33mReblogs: {trending_post['reblogs_count']} \ -Favourites: {trending_post['favourites_count']} \ -Copy: {t_post_url}\033[0m", + f"\033[3;33mReblogs: {trending_post['reblogs_count']} " + f"Favourites: {trending_post['favourites_count']} " + f"Copy: {t_post_url}\033[0m", ) trending_posts_dict[t_post_url] = trending_post return False @@ -217,8 +217,8 @@ async def find_trending_posts( # noqa: C901 remember_to_find_me.pop(fetch_domain) for fetch_domain in remember_to_find_me: - msg = f"Fetching {len(remember_to_find_me[fetch_domain])} \ -less popular posts from {fetch_domain}" + msg = (f"Fetching {len(remember_to_find_me[fetch_domain])} " + f"less popular posts from {fetch_domain}") logging.info(f"\033[1;34m{msg}\033[0m") max_concurrent_requests = 10 semaphore = asyncio.Semaphore(max_concurrent_requests) @@ -383,8 +383,8 @@ def queue_aux_fetch( logging.debug(f"Adding {post_url} to aux_fetches[{parsed_url[0]}]") self.aux_fetches[parsed_url[0]].append((parsed_url, post_url)) logging.debug( - f"aux_fetches[{parsed_url[0]}] is now \ -{self.aux_fetches[parsed_url[0]]}", + f"aux_fetches[{parsed_url[0]}] is now " + f"{self.aux_fetches[parsed_url[0]]}", ) async def do_aux_fetches( @@ -404,7 +404,8 @@ async def fetching_domain( if semaphore is None: semaphore = asyncio.Semaphore(1) async with semaphore: - msg = f"Fetching {len(self.aux_fetches[fetch_domain])} popular posts from {fetch_domain}" + msg = (f"Fetching {len(self.aux_fetches[fetch_domain])} popular posts " + f"from {fetch_domain}") logging.info(f"\033[1;34m{msg}\033[0m") list_of_posts = [] list_of_parsed_urls = [] diff --git a/fedifetcher/getter_wrappers.py b/fedifetcher/getter_wrappers.py index a32a10e6..b2dc8d18 100644 --- a/fedifetcher/getter_wrappers.py +++ b/fedifetcher/getter_wrappers.py @@ -64,8 +64,8 @@ async def get_notification_users( new_notification_users = filter_known_users(notification_users, known_users) logging.info( - f"Found {len(notification_users)} users in notifications, \ -{len(new_notification_users)} of which are new", + f"Found {len(notification_users)} users in notifications, " + f"{len(new_notification_users)} of which are new", ) # return [user.get("account") for user in filter_known_users( @@ -109,8 +109,8 @@ async def get_new_follow_requests( new_follow_requests = filter_known_users(follow_requests, known_followings) logging.info( - f"Got {len(follow_requests)} follow_requests, \ -{len(new_follow_requests)} of which are new", + f"Got {len(follow_requests)} follow_requests, " + f"{len(new_follow_requests)} of which are new", ) return new_follow_requests diff --git a/fedifetcher/helpers/helpers.py b/fedifetcher/helpers/helpers.py index 8e849fd4..e1aea79a 100644 --- a/fedifetcher/helpers/helpers.py +++ b/fedifetcher/helpers/helpers.py @@ -14,9 +14,9 @@ def setup_logging(log_level: str) -> None: logger = logging.getLogger() stdout = colorlog.StreamHandler(stream=sys.stdout) fmt = colorlog.ColoredFormatter( - "%(white)s%(asctime)s%(reset)s | %(log_color)s%(levelname)s%(reset)s | \ -%(threadName)s:%(name)s | %(blue)s%(filename)s:%(lineno)s%(reset)s | %(funcName)s >>> \ -%(log_color)s%(message)s%(reset)s", + "%(white)s%(asctime)s%(reset)s | %(log_color)s%(levelname)s%(reset)s | " + "%(threadName)s:%(name)s | %(blue)s%(filename)s:%(lineno)s%(reset)s | " + "%(funcName)s >>> %(log_color)s%(message)s%(reset)s", ) stdout.setFormatter(fmt) logger.addHandler(stdout) @@ -73,8 +73,8 @@ def get( now = datetime.now(datetime.now(UTC).astimezone().tzinfo) wait = (reset - now).total_seconds() + 1 logging.warning( - f"Rate Limit hit requesting {url} \ -Waiting {wait} sec to retry at {response.headers['x-ratelimit-reset']}", + f"Rate Limit hit requesting {url} Waiting {wait} sec to retry at " + f"{response.headers['x-ratelimit-reset']}", ) time.sleep(wait) return get(url, headers, timeout, max_tries - 1) diff --git a/fedifetcher/main.py b/fedifetcher/main.py index 27db4b27..22ea6c84 100644 --- a/fedifetcher/main.py +++ b/fedifetcher/main.py @@ -71,8 +71,8 @@ async def main(arguments: Namespace) -> None: # noqa: PLR0912, C901, PLR0915 logging.info("Lock file has expired. Removed lock file.") else: logging.info( - f"Lock file age is {datetime.now(UTC) - lock_time} - \ -below --lock-hours={arguments.lock_hours} provided.", + f"Lock file age is {datetime.now(UTC) - lock_time} - " + f"below --lock-hours={arguments.lock_hours} provided.", ) if arguments.on_fail: try: @@ -151,16 +151,15 @@ async def main(arguments: Namespace) -> None: # noqa: PLR0912, C901, PLR0915 ) except Exception: logging.warning( - "Error getting active user IDs. This optional feature \ -requires the admin:read:accounts scope to be enabled on the first access token \ -provided. Continuing without active user IDs.", + "Error getting active user IDs. This optional feature requires the " + "admin:read:accounts scope to be enabled on the first access token " + "provided. Continuing without active user IDs.", ) for _token in arguments.access_token: index = arguments.access_token.index(_token) logging.info( - f"Getting posts for token {index + 1} of \ -{len(arguments.access_token)}", + f"Getting posts for token {index + 1} of {len(arguments.access_token)}", ) await token_posts( _token, @@ -201,8 +200,8 @@ async def main(arguments: Namespace) -> None: # noqa: PLR0912, C901, PLR0915 logging.error(f"Error getting callback url: {ex}") logging.info( - f"\033[1m\033[38;5;208mProcessing finished in \ -\033[38;5;208m{datetime.now(UTC) - start}.\033[0m", + f"\033[1m\033[38;5;208mProcessing finished in " + f"\033[38;5;208m{datetime.now(UTC) - start}.\033[0m", ) except Exception: diff --git a/fedifetcher/mode/token_posts.py b/fedifetcher/mode/token_posts.py index 283d136d..1dc2c524 100644 --- a/fedifetcher/mode/token_posts.py +++ b/fedifetcher/mode/token_posts.py @@ -53,8 +53,8 @@ async def token_posts( # pylint: disable=too-many-arguments # pylint: disable=t # Backfill any post authors, and any mentioned users if arguments.backfill_mentioned_users > 0: logging.info( - f"Backfilling posts from last {arguments.backfill_mentioned_users} \ -mentioned users", + f"Backfilling posts from last {arguments.backfill_mentioned_users} " + f"mentioned users", ) mentioned_users = [] cut_off = datetime.now(datetime.now(UTC).astimezone().tzinfo) - timedelta(